--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:37:47.213694Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-6de22-7f888f584b-85sxv namespace: kserve-ci-e2e-test resourceVersion: "21825" uid: 4b06916f-f4be-4bee-b908-7e7ca985c474 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-6de22-7f888f584b-85sxv to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:37:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:37:47Z" name: ensemble-graph-6de22-7f888f584b-85sxv.18c55787988ab860 namespace: kserve-ci-e2e-test resourceVersion: "21832" uid: fae60652-58ab-47c3-a18e-e623298941ff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:47Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-6de22-7f888f584b-85sxv namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: 4b06916f-f4be-4bee-b908-7e7ca985c474 kind: Event lastTimestamp: "2026-07-24T21:37:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-6de22-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:47Z" name: ensemble-graph-6de22-7f888f584b-85sxv.18c55787a47560ff namespace: kserve-ci-e2e-test resourceVersion: "21846" uid: 999a905c-8304-416b-b115-1220d8b8aa90 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:48Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-6de22-7f888f584b-85sxv namespace: kserve-ci-e2e-test resourceVersion: "21836" uid: 4b06916f-f4be-4bee-b908-7e7ca985c474 kind: Event lastTimestamp: "2026-07-24T21:37:48Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:37:48Z" name: ensemble-graph-6de22-7f888f584b-85sxv.18c55787d5d9b96b namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: e1b6cdba-e0fa-4801-be91-91a3313dd59c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6de22} kind: Pod name: ensemble-graph-6de22-7f888f584b-85sxv namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: 4b06916f-f4be-4bee-b908-7e7ca985c474 kind: Event lastTimestamp: "2026-07-24T21:37:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d388f8d2997cdb455a73f33beea2e1aa8fe959b1d5682a0690a87a2319ba2789" already present on machine metadata: creationTimestamp: "2026-07-24T21:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:48Z" name: ensemble-graph-6de22-7f888f584b-85sxv.18c55787d7287c5d namespace: kserve-ci-e2e-test resourceVersion: "21856" uid: b90424e9-055d-4a0e-bb52-b3c79c201139 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6de22} kind: Pod name: ensemble-graph-6de22-7f888f584b-85sxv namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: 4b06916f-f4be-4bee-b908-7e7ca985c474 kind: Event lastTimestamp: "2026-07-24T21:37:48Z" message: 'Created container: ensemble-graph-6de22' metadata: creationTimestamp: "2026-07-24T21:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:48Z" name: ensemble-graph-6de22-7f888f584b-85sxv.18c55787dabe7351 namespace: kserve-ci-e2e-test resourceVersion: "21857" uid: 8fab52fc-1035-49e4-b8b8-39b34e8f07bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6de22} kind: Pod name: ensemble-graph-6de22-7f888f584b-85sxv namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: 4b06916f-f4be-4bee-b908-7e7ca985c474 kind: Event lastTimestamp: "2026-07-24T21:37:48Z" message: Started container ensemble-graph-6de22 metadata: creationTimestamp: "2026-07-24T21:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:48Z" name: ensemble-graph-6de22-7f888f584b-85sxv.18c55787db4751a5 namespace: kserve-ci-e2e-test resourceVersion: "21858" uid: 619b4f11-df97-40d9-80e5-5b1381712714 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6de22} kind: Pod name: ensemble-graph-6de22-7f888f584b-85sxv namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: 4b06916f-f4be-4bee-b908-7e7ca985c474 kind: Event lastTimestamp: "2026-07-24T21:46:01Z" message: Stopping container ensemble-graph-6de22 metadata: creationTimestamp: "2026-07-24T21:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:01Z" name: ensemble-graph-6de22-7f888f584b-85sxv.18c557fac942a033 namespace: kserve-ci-e2e-test resourceVersion: "25364" uid: 0699274e-0e68-4fc4-8a26-8b004aef698f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:46:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6de22} kind: Pod name: ensemble-graph-6de22-7f888f584b-85sxv namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: 4b06916f-f4be-4bee-b908-7e7ca985c474 kind: Event lastTimestamp: "2026-07-24T21:46:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-24T21:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:29Z" name: ensemble-graph-6de22-7f888f584b-85sxv.18c557fb7b1b0254 namespace: kserve-ci-e2e-test resourceVersion: "25999" uid: 7c4496ca-380d-4049-bb85-7f6f6a5e2acf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-6de22-7f888f584b namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: a91dcb63-8adc-4612-8247-79ebbc84f6aa kind: Event lastTimestamp: "2026-07-24T21:37:47Z" message: 'Created pod: ensemble-graph-6de22-7f888f584b-85sxv' metadata: creationTimestamp: "2026-07-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:47Z" name: ensemble-graph-6de22-7f888f584b.18c5578797c06685 namespace: kserve-ci-e2e-test resourceVersion: "21829" uid: cf9a4750-4c9e-4d99-af75-712fc52da087 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-6de22 namespace: kserve-ci-e2e-test resourceVersion: "21818" uid: d3999081-9a55-4f18-a3d9-1601787bf57e kind: Event lastTimestamp: "2026-07-24T21:37:47Z" message: Scaled up replica set ensemble-graph-6de22-7f888f584b from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:47Z" name: ensemble-graph-6de22.18c5578796ccf476 namespace: kserve-ci-e2e-test resourceVersion: "21821" uid: a11bd9fe-3cbe-4e07-a515-59b19b53a9ad reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-6de22 namespace: kserve-ci-e2e-test resourceVersion: "21839" uid: 2d4e9520-323a-4f53-97ed-01483bb1cf8f kind: Event lastTimestamp: "2026-07-24T21:37:47Z" message: InferenceGraph [ensemble-graph-6de22] is Ready metadata: creationTimestamp: "2026-07-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:47Z" name: ensemble-graph-6de22.18c557879cee9cbb namespace: kserve-ci-e2e-test resourceVersion: "21841" uid: 14960c7e-566b-4b3b-9756-9dd855531568 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:38:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-6de22 namespace: kserve-ci-e2e-test resourceVersion: "21819" uid: 81d119a5-574c-4dd6-a67d-dfc9dea4e740 kind: Event lastTimestamp: "2026-07-24T21:38:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:17Z" name: ensemble-graph-6de22.18c5578b152e771d namespace: kserve-ci-e2e-test resourceVersion: "22096" uid: bb123e7e-845b-4455-a7d2-056f9063db9d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:38:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-6de22 namespace: kserve-ci-e2e-test resourceVersion: "21819" uid: 81d119a5-574c-4dd6-a67d-dfc9dea4e740 kind: Event lastTimestamp: "2026-07-24T21:38:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:17Z" name: ensemble-graph-6de22.18c5578b159606ff namespace: kserve-ci-e2e-test resourceVersion: "22098" uid: 454eef60-46a4-44d2-82ee-2dfa2f9e2257 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:36:36.881979Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8 namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 9d8b9fda-c439-4bc6-8ab6-0d59d683fefb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-b6aba-6bbf9ddf75-gt6b8 to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:36:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:36:36Z" name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8.18c557773871f44e namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: a7a5fe69-c24c-4741-b364-62b6d133def3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:37Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 9d8b9fda-c439-4bc6-8ab6-0d59d683fefb kind: Event lastTimestamp: "2026-07-24T21:36:37Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:36:37Z" name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8.18c557775207a2a1 namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: ba2d36b9-a781-4f70-bbd1-d9e9c24bd312 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b6aba} kind: Pod name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 9d8b9fda-c439-4bc6-8ab6-0d59d683fefb kind: Event lastTimestamp: "2026-07-24T21:36:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d388f8d2997cdb455a73f33beea2e1aa8fe959b1d5682a0690a87a2319ba2789" already present on machine metadata: creationTimestamp: "2026-07-24T21:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:37Z" name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8.18c5577754cc1092 namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: 6c132817-839b-4a89-a1fc-e52f36e71570 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b6aba} kind: Pod name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 9d8b9fda-c439-4bc6-8ab6-0d59d683fefb kind: Event lastTimestamp: "2026-07-24T21:36:37Z" message: 'Created container: ensemble-graph-b6aba' metadata: creationTimestamp: "2026-07-24T21:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:37Z" name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8.18c5577758294aee namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: d152baaf-7662-44f3-9648-b338bee22db2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b6aba} kind: Pod name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 9d8b9fda-c439-4bc6-8ab6-0d59d683fefb kind: Event lastTimestamp: "2026-07-24T21:36:37Z" message: Started container ensemble-graph-b6aba metadata: creationTimestamp: "2026-07-24T21:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:37Z" name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8.18c55777589f95d0 namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 8180933d-81aa-4633-abb6-df637962707f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b6aba} kind: Pod name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 9d8b9fda-c439-4bc6-8ab6-0d59d683fefb kind: Event lastTimestamp: "2026-07-24T21:36:46Z" message: Stopping container ensemble-graph-b6aba metadata: creationTimestamp: "2026-07-24T21:36:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:46Z" name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8.18c5577990586e98 namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 82070116-04a9-4637-85ac-a5913dce4259 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:36:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b6aba} kind: Pod name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 9d8b9fda-c439-4bc6-8ab6-0d59d683fefb kind: Event lastTimestamp: "2026-07-24T21:37:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-24T21:36:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:13Z" name: ensemble-graph-b6aba-6bbf9ddf75-gt6b8.18c55779f294afb1 namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: f3568771-d9df-4831-a373-147775867f13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-b6aba-6bbf9ddf75 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 0a09d01a-a7af-4277-b058-9cff37d12e3a kind: Event lastTimestamp: "2026-07-24T21:36:36Z" message: 'Created pod: ensemble-graph-b6aba-6bbf9ddf75-gt6b8' metadata: creationTimestamp: "2026-07-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:36Z" name: ensemble-graph-b6aba-6bbf9ddf75.18c5577737842310 namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: 6b225fc8-d84e-4397-97a1-2afe4e4328b7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-b6aba namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: 9037d5e0-e9b9-4783-a530-f0917b089d13 kind: Event lastTimestamp: "2026-07-24T21:36:36Z" message: Scaled up replica set ensemble-graph-b6aba-6bbf9ddf75 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:36:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:36Z" name: ensemble-graph-b6aba.18c55777369d44e1 namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: 8c5bfa35-6e7a-4a01-b3e9-9e0be25cd5b9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-b6aba namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 7658dcd7-047e-4388-848a-7b630195d662 kind: Event lastTimestamp: "2026-07-24T21:36:37Z" message: InferenceGraph [ensemble-graph-b6aba] is Ready metadata: creationTimestamp: "2026-07-24T21:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:36:37Z" name: ensemble-graph-b6aba.18c557773fed31bf namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: c27486cd-48f3-4260-8d4b-bcbf1cf29f00 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:37:37.209552Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz namespace: kserve-ci-e2e-test resourceVersion: "21682" uid: c21056da-69b4-4ad2-bd14-a65e99a848ed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz.18c55785443fb271 namespace: kserve-ci-e2e-test resourceVersion: "21688" uid: 7e1b6082-1318-4aec-bedf-b8dc6311fc43 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz namespace: kserve-ci-e2e-test resourceVersion: "21695" uid: c21056da-69b4-4ad2-bd14-a65e99a848ed kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz.18c557855dcad900 namespace: kserve-ci-e2e-test resourceVersion: "21708" uid: 632a6a70-7c56-48d8-9251-42a9508fe6a5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: c21056da-69b4-4ad2-bd14-a65e99a848ed kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz.18c557855f43b2b4 namespace: kserve-ci-e2e-test resourceVersion: "21710" uid: 2343b805-9026-4246-8084-c008557cd2c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: c21056da-69b4-4ad2-bd14-a65e99a848ed kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz.18c5578562258094 namespace: kserve-ci-e2e-test resourceVersion: "21713" uid: 43e8c508-8217-4ba2-99d0-e6675a791e15 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: c21056da-69b4-4ad2-bd14-a65e99a848ed kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz.18c5578562a7416d namespace: kserve-ci-e2e-test resourceVersion: "21714" uid: 18790474-5ede-40b4-93d1-b45621eddda5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: c21056da-69b4-4ad2-bd14-a65e99a848ed kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz.18c5578562b0ad7a namespace: kserve-ci-e2e-test resourceVersion: "21715" uid: 7e58e13d-f734-467d-a118-6287087197e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: c21056da-69b4-4ad2-bd14-a65e99a848ed kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz.18c55785657c0734 namespace: kserve-ci-e2e-test resourceVersion: "21717" uid: 4b695634-0dd8-4a8f-92ce-b6bdd673986f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: c21056da-69b4-4ad2-bd14-a65e99a848ed kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz.18c5578565ed8d3d namespace: kserve-ci-e2e-test resourceVersion: "21718" uid: 68c2d2e3-da1a-4f5f-8cde-5da01b9ed8f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:37:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: c21056da-69b4-4ad2-bd14-a65e99a848ed kind: Event lastTimestamp: "2026-07-24T21:38:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:38:14Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz.18c55785a7f2f1df namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: 3004cc8b-e8f3-4a1a-8b76-bb3284d35c9a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: c21056da-69b4-4ad2-bd14-a65e99a848ed kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz.18c5580676714e46 namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: 0541cc72-9845-48e4-92a6-9c09d246ff17 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: c21056da-69b4-4ad2-bd14-a65e99a848ed kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz.18c5580676720307 namespace: kserve-ci-e2e-test resourceVersion: "26277" uid: 65d9605c-9856-4707-abd9-f4dc05e12c97 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: c21056da-69b4-4ad2-bd14-a65e99a848ed kind: Event lastTimestamp: "2026-07-24T21:46:55Z" message: 'Readiness probe failed: Get "https://10.132.0.42:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-24T21:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:55Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz.18c5580757ed0d0f namespace: kserve-ci-e2e-test resourceVersion: "26364" uid: 8fa00e7e-fd50-4a32-96f1-ff43a67a10b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: c21056da-69b4-4ad2-bd14-a65e99a848ed kind: Event lastTimestamp: "2026-07-24T21:46:55Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: i/o timeout' metadata: creationTimestamp: "2026-07-24T21:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:55Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz.18c55807583b77c4 namespace: kserve-ci-e2e-test resourceVersion: "26365" uid: b32e4e0b-63e6-4596-8d05-55c72e42dfd1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-59fd1-predictor-6c56cf6df7 namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: 8fee9e13-9f03-40f4-b50b-476089023334 kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: 'Created pod: error-404-isvc-59fd1-predictor-6c56cf6df7-pknkz' metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-59fd1-predictor-6c56cf6df7.18c5578543d73a4a namespace: kserve-ci-e2e-test resourceVersion: "21685" uid: ee799a65-bd66-43a8-8715-60e180f6af9e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-59fd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: cf6dc062-6690-4637-a9d1-5fb7bda75e3b kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Scaled up replica set error-404-isvc-59fd1-predictor-6c56cf6df7 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-59fd1-predictor.18c557854219a8f1 namespace: kserve-ci-e2e-test resourceVersion: "21676" uid: 839757e2-4aa4-408e-8075-aa00343be8c0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:37:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-59fd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: 687a4d6d-1a53-4c77-8e98-52313686959d kind: Event lastTimestamp: "2026-07-24T21:38:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:37Z" name: error-404-isvc-59fd1-predictor.18c55788c0854faf namespace: kserve-ci-e2e-test resourceVersion: "22278" uid: c33318a4-5cef-466b-8c91-a2b85e3df580 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:37:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-59fd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: 687a4d6d-1a53-4c77-8e98-52313686959d kind: Event lastTimestamp: "2026-07-24T21:38:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:37Z" name: error-404-isvc-59fd1-predictor.18c55788c0f16017 namespace: kserve-ci-e2e-test resourceVersion: "22282" uid: 0fba08d7-64cc-43aa-8187-0acdc4478eab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:38:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-59fd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21895" uid: 687a4d6d-1a53-4c77-8e98-52313686959d kind: Event lastTimestamp: "2026-07-24T21:38:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:38:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:07Z" name: error-404-isvc-59fd1-predictor.18c5578c40150398 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 63bfc28f-6728-4ed3-96fc-99467a57a5fc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:38:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-59fd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21895" uid: 687a4d6d-1a53-4c77-8e98-52313686959d kind: Event lastTimestamp: "2026-07-24T21:38:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:38:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:07Z" name: error-404-isvc-59fd1-predictor.18c5578c409192fd namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: 1ef9747e-036f-4c43-be30-8f72bba8787f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-59fd1 namespace: kserve-ci-e2e-test resourceVersion: "21656" uid: 9bccf5db-4e91-46cd-b447-49362b1709d2 kind: Event lastTimestamp: "2026-07-24T21:38:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:38:25Z" name: error-404-isvc-59fd1.18c5578542206f90 namespace: kserve-ci-e2e-test resourceVersion: "22169" uid: 9e978ff4-2e04-4349-b892-71abc726cffb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-59fd1 namespace: kserve-ci-e2e-test resourceVersion: "21656" uid: 9bccf5db-4e91-46cd-b447-49362b1709d2 kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-59fd1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-59fd1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-59fd1.18c5578542af2546 namespace: kserve-ci-e2e-test resourceVersion: "21679" uid: dcc86aa1-3cdc-49a8-95e4-982a11b43c5f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-59fd1 namespace: kserve-ci-e2e-test resourceVersion: "21656" uid: 9bccf5db-4e91-46cd-b447-49362b1709d2 kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-59fd1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-59fd1.18c5578542af6fde namespace: kserve-ci-e2e-test resourceVersion: "21680" uid: 4cd29645-2f61-4486-88da-2ba3d957a81d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:38:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-59fd1 namespace: kserve-ci-e2e-test resourceVersion: "22166" uid: 9bccf5db-4e91-46cd-b447-49362b1709d2 kind: Event lastTimestamp: "2026-07-24T21:38:25Z" message: InferenceService [error-404-isvc-59fd1] is Ready metadata: creationTimestamp: "2026-07-24T21:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:38:25Z" name: error-404-isvc-59fd1.18c557906616484b namespace: kserve-ci-e2e-test resourceVersion: "22167" uid: 9a97b15d-9076-41f4-813a-f6adf2ef80dd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:36:47.324883Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245 namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: 8782f4dd-7890-4128-aa45-087f880965a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6de22-predictor-6c4f986f4f-dx245 to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:36:47Z" name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245.18c55779a6e4060b namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: fb2e5769-dcb0-483a-b2eb-2a179509e9f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 8782f4dd-7890-4128-aa45-087f880965a6 kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6de22-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:47Z" name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245.18c55779b5a89e79 namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: 58da679e-20c3-4df0-be35-8e4a2381320a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245 namespace: kserve-ci-e2e-test resourceVersion: "21073" uid: 8782f4dd-7890-4128-aa45-087f880965a6 kind: Event lastTimestamp: "2026-07-24T21:36:48Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:36:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:36:48Z" name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245.18c55779e41a04da namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 8d68f38f-5296-41f5-a80b-09e98dda8600 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 8782f4dd-7890-4128-aa45-087f880965a6 kind: Event lastTimestamp: "2026-07-24T21:36:48Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:36:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:48Z" name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245.18c55779e68083b3 namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: 178d0ed6-51d9-4f3f-9397-af4ae05193e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 8782f4dd-7890-4128-aa45-087f880965a6 kind: Event lastTimestamp: "2026-07-24T21:36:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:36:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:48Z" name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245.18c55779e989bd83 namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: af0b621f-aa3f-4e21-b016-253ba773475e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 8782f4dd-7890-4128-aa45-087f880965a6 kind: Event lastTimestamp: "2026-07-24T21:36:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:36:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:48Z" name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245.18c55779e9ffa8e1 namespace: kserve-ci-e2e-test resourceVersion: "21104" uid: 35a0f9fb-aa72-4c0c-9e85-7fb59dc0f4f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 8782f4dd-7890-4128-aa45-087f880965a6 kind: Event lastTimestamp: "2026-07-24T21:36:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:36:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:48Z" name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245.18c55779ea081e7f namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: ce7bee58-4016-4887-9903-471195641334 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 8782f4dd-7890-4128-aa45-087f880965a6 kind: Event lastTimestamp: "2026-07-24T21:36:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:36:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:48Z" name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245.18c55779ed4c6766 namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: 69143b75-d80b-4446-89ae-4b68f1417104 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 8782f4dd-7890-4128-aa45-087f880965a6 kind: Event lastTimestamp: "2026-07-24T21:36:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:36:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:48Z" name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245.18c55779edc0618b namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: a0ced82b-c950-4209-a5c1-27a8f1654a09 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-24T21:36:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 8782f4dd-7890-4128-aa45-087f880965a6 kind: Event lastTimestamp: "2026-07-24T21:46:04Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:36:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:04Z" name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245.18c55779f557d481 namespace: kserve-ci-e2e-test resourceVersion: "25738" uid: 542cd5af-2149-440a-8eec-36fb79a9f038 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 8782f4dd-7890-4128-aa45-087f880965a6 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245.18c557fae0dea8ce namespace: kserve-ci-e2e-test resourceVersion: "25503" uid: f3d8f016-00d7-4fb8-8d69-67e7b2ca4cdf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 8782f4dd-7890-4128-aa45-087f880965a6 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245.18c557fae0dfde88 namespace: kserve-ci-e2e-test resourceVersion: "25504" uid: 65e0368b-3f99-40d4-9c4b-cf53fb959695 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 8782f4dd-7890-4128-aa45-087f880965a6 kind: Event lastTimestamp: "2026-07-24T21:46:04Z" message: 'Readiness probe failed: Get "https://10.132.0.40:8643/healthz": dial tcp 10.132.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:04Z" name: error-404-isvc-6de22-predictor-6c4f986f4f-dx245.18c557fb69c2b9cc namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: f23c4cf9-3109-4590-be7a-7d5fc4bee0cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6de22-predictor-6c4f986f4f namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 70980752-c0dc-4569-bbbc-abf128074a4d kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: 'Created pod: error-404-isvc-6de22-predictor-6c4f986f4f-dx245' metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:47Z" name: error-404-isvc-6de22-predictor-6c4f986f4f.18c55779a63f3ceb namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: 0719f346-dbcc-4af4-abca-debc71f5ad7b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6de22-predictor namespace: kserve-ci-e2e-test resourceVersion: "21051" uid: 958f181e-3a6e-4dc2-8d1e-a417de33ea7f kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: Scaled up replica set error-404-isvc-6de22-predictor-6c4f986f4f from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:47Z" name: error-404-isvc-6de22-predictor.18c55779a3fdd6f3 namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: a00a7e40-4862-4349-9f4c-33fb8ddb54d3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:37:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6de22-predictor namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: bbec8401-b7b7-4d38-9120-b472c2012810 kind: Event lastTimestamp: "2026-07-24T21:37:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:32Z" name: error-404-isvc-6de22-predictor.18c5577d22f6f50d namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: ee4bd1ea-dcd6-4238-8e72-a91e6f97746a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:37:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6de22-predictor namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: bbec8401-b7b7-4d38-9120-b472c2012810 kind: Event lastTimestamp: "2026-07-24T21:37:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:32Z" name: error-404-isvc-6de22-predictor.18c5577d2373f966 namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: c30ee0df-e1e7-4ecf-a9a5-be2f9c01aca9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:37:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6de22-predictor namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: bbec8401-b7b7-4d38-9120-b472c2012810 kind: Event lastTimestamp: "2026-07-24T21:38:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:37:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:02Z" name: error-404-isvc-6de22-predictor.18c55780a2a4447a namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: 03af3d6c-c42c-4189-9088-b0ea4ae4701e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:37:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6de22-predictor namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: bbec8401-b7b7-4d38-9120-b472c2012810 kind: Event lastTimestamp: "2026-07-24T21:38:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:37:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:02Z" name: error-404-isvc-6de22-predictor.18c55780a2f8062a namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: 905b8e1f-6b52-4529-80eb-e1a14a99c017 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6de22 namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 57b436f3-8d98-4362-8f35-9d4f004ef662 kind: Event lastTimestamp: "2026-07-24T21:37:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:34Z" name: error-404-isvc-6de22.18c55779a416375d namespace: kserve-ci-e2e-test resourceVersion: "21569" uid: 53b89e5c-0a44-4125-a17a-4405fe291fa8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6de22 namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 57b436f3-8d98-4362-8f35-9d4f004ef662 kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: 'Failed to update status for InferenceService "error-404-isvc-6de22": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6de22": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:36:47Z" name: error-404-isvc-6de22.18c55779a4b847dc namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 17297ecd-36ed-4c45-8ad7-6f567184e675 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6de22 namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 57b436f3-8d98-4362-8f35-9d4f004ef662 kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6de22": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:36:47Z" name: error-404-isvc-6de22.18c55779a4b8a6af namespace: kserve-ci-e2e-test resourceVersion: "21062" uid: bba0612f-ea48-4a17-9f88-5d234f657c2f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6de22 namespace: kserve-ci-e2e-test resourceVersion: "21565" uid: 57b436f3-8d98-4362-8f35-9d4f004ef662 kind: Event lastTimestamp: "2026-07-24T21:37:34Z" message: InferenceService [error-404-isvc-6de22] is Ready metadata: creationTimestamp: "2026-07-24T21:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:34Z" name: error-404-isvc-6de22.18c55784b17dcaf9 namespace: kserve-ci-e2e-test resourceVersion: "21567" uid: 60b9de17-e722-45fa-b696-5b0f186e169d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:24:51.905270Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:24:51Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556d3148f0540 namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: 40064ea1-dbc0-4359-8f8c-424166b1aad4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: "2026-07-24T21:24:52Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:24:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:24:52Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556d32e5659a5 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: 03871062-2151-40ad-a0f4-ede2d625f92c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: "2026-07-24T21:24:52Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1716" metadata: creationTimestamp: "2026-07-24T21:24:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:52Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556d32fec7da6 namespace: kserve-ci-e2e-test resourceVersion: "14788" uid: 5fdce7c8-baaf-4233-a2ef-5ac00de2f400 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: "2026-07-24T21:25:03Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1716" in 11.394s (11.394s including waiting). Image size: 1333940831 bytes.' metadata: creationTimestamp: "2026-07-24T21:25:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:03Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556d5d716abfe namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 5ae130ae-2235-4849-bdbf-803d9dec2011 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: "2026-07-24T21:25:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:25:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:03Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556d5da17c318 namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: b234d1af-4050-46d0-b196-31e763032141 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: "2026-07-24T21:25:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:25:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:03Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556d5da8cc2ce namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 58ff2e8d-ee84-4229-ad3f-2e9b7f13abd1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: "2026-07-24T21:25:03Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-24T21:25:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:03Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556d5da97b02b namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: 73cfee75-7c1b-4873-9ca9-c1a8c88f7c7f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: "2026-07-24T21:25:06Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.25s (2.25s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-24T21:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:06Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556d660c0a0cd namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: d377068b-c3df-44e8-ab6c-5a59b8ba2368 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: "2026-07-24T21:25:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:06Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556d66399cc6b namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: a1ca4fe1-8ce4-40de-96d5-c75aefd3ab92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: "2026-07-24T21:25:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:06Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556d6640dae57 namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: d6e8f2dc-445b-4faf-ae49-088ac92a9e77 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-24T21:25:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: "2026-07-24T21:26:23Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:25:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:23Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556d6ac57b997 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 50db27c1-2c37-40d8-8171-c4d54c76dcf3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: "2026-07-24T21:26:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:21Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556e807479ba0 namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 2d02075b-01c2-4e11-bdcd-336be8dc8de9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: "2026-07-24T21:26:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:21Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556e807481e87 namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: c60f4ab9-7ee2-401c-8e47-c76f712f87f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 6ff6f332-3d37-402a-a5b4-d35b75b85475 kind: Event lastTimestamp: "2026-07-24T21:26:23Z" message: 'Readiness probe failed: Get "https://10.132.0.30:8643/healthz": dial tcp 10.132.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:26:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:23Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq.18c556e85e8c8e66 namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: 9b3f098b-3c89-4c22-ab26-748e5e078918 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-77ecc-predictor-7bcd8f7bd9 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 693c255a-acc5-41a1-8dae-1cfe4622d21c kind: Event lastTimestamp: "2026-07-24T21:24:51Z" message: 'Created pod: error-404-isvc-77ecc-predictor-7bcd8f7bd9-vdcnq' metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:24:51Z" name: error-404-isvc-77ecc-predictor-7bcd8f7bd9.18c556d313c32c41 namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: 14dc1920-fe3e-499e-b76b-373eaa864290 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-77ecc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14702" uid: 4bc5ad63-ca19-432a-b9eb-dab27d4b7449 kind: Event lastTimestamp: "2026-07-24T21:24:51Z" message: Scaled up replica set error-404-isvc-77ecc-predictor-7bcd8f7bd9 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:24:51Z" name: error-404-isvc-77ecc-predictor.18c556d311e4a13f namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: d053927a-8b13-41da-af49-799d7ee4dfc6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:25:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-77ecc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: e0ca8263-be92-44a9-a33d-ab729a150aae kind: Event lastTimestamp: "2026-07-24T21:25:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:25:36Z" name: error-404-isvc-77ecc-predictor.18c556d69046f55c namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: b593010e-ac41-4bc2-8db5-cf9f0eddb10b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:25:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-77ecc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: e0ca8263-be92-44a9-a33d-ab729a150aae kind: Event lastTimestamp: "2026-07-24T21:25:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:25:36Z" name: error-404-isvc-77ecc-predictor.18c556d690a99be4 namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: ec52bd31-b5f2-42e0-a11b-6189b284e341 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:25:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-77ecc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: e0ca8263-be92-44a9-a33d-ab729a150aae kind: Event lastTimestamp: "2026-07-24T21:26:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:06Z" name: error-404-isvc-77ecc-predictor.18c556da0f6b3a4d namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 6ba2f603-d0b6-474c-ba73-ebff20f630c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:25:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-77ecc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: e0ca8263-be92-44a9-a33d-ab729a150aae kind: Event lastTimestamp: "2026-07-24T21:26:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:06Z" name: error-404-isvc-77ecc-predictor.18c556da0fc42588 namespace: kserve-ci-e2e-test resourceVersion: "15508" uid: 6ba33d8c-abaf-4e8e-b1a3-974403cc27c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-77ecc namespace: kserve-ci-e2e-test resourceVersion: "14687" uid: ad21c61b-76b6-470f-985d-f3bd1245ce92 kind: Event lastTimestamp: "2026-07-24T21:25:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:25:53Z" name: error-404-isvc-77ecc.18c556d311df6427 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: 9e01af1d-c24e-48dd-9693-e1d645a051e0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-77ecc namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: ad21c61b-76b6-470f-985d-f3bd1245ce92 kind: Event lastTimestamp: "2026-07-24T21:25:53Z" message: InferenceService [error-404-isvc-77ecc] is Ready metadata: creationTimestamp: "2026-07-24T21:25:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:25:53Z" name: error-404-isvc-77ecc.18c556e1680a7fc9 namespace: kserve-ci-e2e-test resourceVersion: "15395" uid: 1193e767-2a02-429b-849e-03adc4a38083 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:46:02.286753Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh namespace: kserve-ci-e2e-test resourceVersion: "25477" uid: 1d56fde1-16b4-4576-9de3-2ac4af98b1aa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh.18c557fadd321a20 namespace: kserve-ci-e2e-test resourceVersion: "25481" uid: 35c08261-9747-464a-bf3f-95a4a297297a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh namespace: kserve-ci-e2e-test resourceVersion: "25490" uid: 1d56fde1-16b4-4576-9de3-2ac4af98b1aa kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh.18c557faf6de4119 namespace: kserve-ci-e2e-test resourceVersion: "25517" uid: 66f3eeff-45c3-4d9f-a2b0-845dc0051252 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 1d56fde1-16b4-4576-9de3-2ac4af98b1aa kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh.18c557faf82af87d namespace: kserve-ci-e2e-test resourceVersion: "25521" uid: 5bdea33f-9efa-4fa2-8ea7-918ee555c0a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 1d56fde1-16b4-4576-9de3-2ac4af98b1aa kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh.18c557fafbf58144 namespace: kserve-ci-e2e-test resourceVersion: "25525" uid: a9d0d08b-5ef7-4543-8940-3c04819d1792 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 1d56fde1-16b4-4576-9de3-2ac4af98b1aa kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh.18c557fafc689694 namespace: kserve-ci-e2e-test resourceVersion: "25526" uid: 525f749c-1729-4f7f-bba8-c10a2217c763 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 1d56fde1-16b4-4576-9de3-2ac4af98b1aa kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh.18c557fafc717aca namespace: kserve-ci-e2e-test resourceVersion: "25527" uid: e8dd0498-2191-4f69-a642-2dd5c2869273 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 1d56fde1-16b4-4576-9de3-2ac4af98b1aa kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh.18c557faff488497 namespace: kserve-ci-e2e-test resourceVersion: "25528" uid: 047af35e-daeb-45f5-a211-b53282c76bca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 1d56fde1-16b4-4576-9de3-2ac4af98b1aa kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh.18c557faffbcb04e namespace: kserve-ci-e2e-test resourceVersion: "25529" uid: c1afef3c-6b42-4c53-8615-0cadd861fc0a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:46:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 1d56fde1-16b4-4576-9de3-2ac4af98b1aa kind: Event lastTimestamp: "2026-07-24T21:46:40Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:46:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:40Z" name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh.18c557fb51fca6dd namespace: kserve-ci-e2e-test resourceVersion: "26082" uid: c3c4e6ff-616e-41be-a3d7-278c5a0449d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 1d56fde1-16b4-4576-9de3-2ac4af98b1aa kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:12Z" name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh.18c5580b2e3e56a2 namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 7bb6faba-06c7-41fb-b6be-623d3b607055 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 1d56fde1-16b4-4576-9de3-2ac4af98b1aa kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:12Z" name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh.18c5580b2e3ed08f namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 67aa28db-f1a1-4d10-b494-72eb2c88c087 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh namespace: kserve-ci-e2e-test resourceVersion: "25479" uid: 1d56fde1-16b4-4576-9de3-2ac4af98b1aa kind: Event lastTimestamp: "2026-07-24T21:47:15Z" message: 'Readiness probe failed: Get "https://10.132.0.45:8643/healthz": dial tcp 10.132.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:15Z" name: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh.18c5580bda3ee593 namespace: kserve-ci-e2e-test resourceVersion: "26706" uid: 26fee94d-3304-4604-9889-15bedc97d9fd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9739a-predictor-68dc85c8c7 namespace: kserve-ci-e2e-test resourceVersion: "25470" uid: 3734077a-64f4-451a-9430-b746987c4804 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: 'Created pod: error-404-isvc-9739a-predictor-68dc85c8c7-jj5rh' metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-9739a-predictor-68dc85c8c7.18c557fadc89a291 namespace: kserve-ci-e2e-test resourceVersion: "25478" uid: 302340ac-7887-4085-be2f-1860a98524f6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9739a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25467" uid: 0a8d8138-f8c8-4272-b124-262b54b7b0db kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Scaled up replica set error-404-isvc-9739a-predictor-68dc85c8c7 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-9739a-predictor.18c557fadab47a9c namespace: kserve-ci-e2e-test resourceVersion: "25471" uid: 10608ac0-de7d-4b8b-b076-72641f2fad60 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:46:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9739a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25469" uid: 9e343b48-ba13-4e32-a095-a868bcbcab78 kind: Event lastTimestamp: "2026-07-24T21:46:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:46:47Z" name: error-404-isvc-9739a-predictor.18c557fe596c7728 namespace: kserve-ci-e2e-test resourceVersion: "26137" uid: c4dba49d-7196-4704-864a-95427700ba20 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:46:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9739a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25469" uid: 9e343b48-ba13-4e32-a095-a868bcbcab78 kind: Event lastTimestamp: "2026-07-24T21:46:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:46:47Z" name: error-404-isvc-9739a-predictor.18c557fe59e88891 namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 76d25a0f-3b1f-4690-a33c-0dc28da2ffa6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9739a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: 9e343b48-ba13-4e32-a095-a868bcbcab78 kind: Event lastTimestamp: "2026-07-24T21:47:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:02Z" name: error-404-isvc-9739a-predictor.18c55808d642fc6f namespace: kserve-ci-e2e-test resourceVersion: "26440" uid: 020b61a9-82ed-4131-b706-9ccefaf60956 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9739a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: 9e343b48-ba13-4e32-a095-a868bcbcab78 kind: Event lastTimestamp: "2026-07-24T21:47:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:02Z" name: error-404-isvc-9739a-predictor.18c55808d6ca23d6 namespace: kserve-ci-e2e-test resourceVersion: "26442" uid: d45c5e7a-9b5e-4a1e-8477-5ac9979351a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9739a namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 5197c977-1e23-4946-bdd0-c0ab249d278e kind: Event lastTimestamp: "2026-07-24T21:46:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:46:50Z" name: error-404-isvc-9739a.18c557fadadc5657 namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: 805b4eef-291e-4e54-a074-8dd378096947 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9739a namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 5197c977-1e23-4946-bdd0-c0ab249d278e kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: 'Failed to update status for InferenceService "error-404-isvc-9739a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9739a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-9739a.18c557fadb72b52c namespace: kserve-ci-e2e-test resourceVersion: "25474" uid: 92f4fc97-d862-4c2f-a6e1-e72652a7fd47 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9739a namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 5197c977-1e23-4946-bdd0-c0ab249d278e kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9739a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:46:02Z" name: error-404-isvc-9739a.18c557fadb72f032 namespace: kserve-ci-e2e-test resourceVersion: "25476" uid: f2abbcdf-215c-448f-bb7c-f1197908c056 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9739a namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: 5197c977-1e23-4946-bdd0-c0ab249d278e kind: Event lastTimestamp: "2026-07-24T21:46:50Z" message: InferenceService [error-404-isvc-9739a] is Ready metadata: creationTimestamp: "2026-07-24T21:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:46:50Z" name: error-404-isvc-9739a.18c558060db34494 namespace: kserve-ci-e2e-test resourceVersion: "26172" uid: 800888d7-0742-49eb-8a67-5eae3963406e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:46:52.133387Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2 namespace: kserve-ci-e2e-test resourceVersion: "26281" uid: 40d3f802-7324-4f66-b47e-4a3ba18fe0aa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a916f-predictor-5548f55f77-vn8q2 to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2.18c55806784968a1 namespace: kserve-ci-e2e-test resourceVersion: "26285" uid: e587bd75-a353-4f33-9031-4c0ebc744b2c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2 namespace: kserve-ci-e2e-test resourceVersion: "26290" uid: 40d3f802-7324-4f66-b47e-4a3ba18fe0aa kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2.18c5580691e128d9 namespace: kserve-ci-e2e-test resourceVersion: "26306" uid: b8f095b2-3541-4ced-b448-f3ed2afb6f8d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2 namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: 40d3f802-7324-4f66-b47e-4a3ba18fe0aa kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2.18c558069332910a namespace: kserve-ci-e2e-test resourceVersion: "26309" uid: 0a84cae1-8f7e-418a-bbf2-db5baca7806e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2 namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: 40d3f802-7324-4f66-b47e-4a3ba18fe0aa kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2.18c5580696f3cdc5 namespace: kserve-ci-e2e-test resourceVersion: "26310" uid: c1fd7bba-55df-478a-acd2-4e7fabda9b3c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2 namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: 40d3f802-7324-4f66-b47e-4a3ba18fe0aa kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2.18c558069765f57b namespace: kserve-ci-e2e-test resourceVersion: "26311" uid: dbe6b050-9b2c-45cc-b08c-d268d1380bbc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2 namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: 40d3f802-7324-4f66-b47e-4a3ba18fe0aa kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2.18c5580697701f35 namespace: kserve-ci-e2e-test resourceVersion: "26312" uid: 07378673-f8a7-4cbc-8423-571ee875a6d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2 namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: 40d3f802-7324-4f66-b47e-4a3ba18fe0aa kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2.18c558069a663f63 namespace: kserve-ci-e2e-test resourceVersion: "26313" uid: fe065525-3965-4c12-b8f7-b6f4b762c1b1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2 namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: 40d3f802-7324-4f66-b47e-4a3ba18fe0aa kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2.18c558069adc432a namespace: kserve-ci-e2e-test resourceVersion: "26314" uid: f99b6eed-e069-4410-ba90-96bf154545d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2 namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: 40d3f802-7324-4f66-b47e-4a3ba18fe0aa kind: Event lastTimestamp: "2026-07-24T21:47:30Z" message: 'Readiness probe failed: dial tcp 10.132.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:30Z" name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2.18c5580704e1e18c namespace: kserve-ci-e2e-test resourceVersion: "26840" uid: 0e8929d5-6309-4092-86b7-b8ba7bf22689 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T22:04:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2 namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: 40d3f802-7324-4f66-b47e-4a3ba18fe0aa kind: Event lastTimestamp: "2026-07-24T22:04:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T22:04:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T22:04:11Z" name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2.18c558f87bb9ed75 namespace: kserve-ci-e2e-test resourceVersion: "33975" uid: 17c3bb8c-4e7b-488a-995e-be721a5c257c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T22:04:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2 namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: 40d3f802-7324-4f66-b47e-4a3ba18fe0aa kind: Event lastTimestamp: "2026-07-24T22:04:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T22:04:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T22:04:11Z" name: error-404-isvc-a916f-predictor-5548f55f77-vn8q2.18c558f87bba50a5 namespace: kserve-ci-e2e-test resourceVersion: "33976" uid: 625b8caa-7ddc-4d8d-821a-32a78fe2f867 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a916f-predictor-5548f55f77 namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: ad7b0c4b-b3bb-49d3-8c59-d78a0057b581 kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: 'Created pod: error-404-isvc-a916f-predictor-5548f55f77-vn8q2' metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-a916f-predictor-5548f55f77.18c5580677d919dd namespace: kserve-ci-e2e-test resourceVersion: "26282" uid: f9e0d812-eb72-4a01-bc10-6cae8abe1410 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a916f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26263" uid: 67fd56ca-e38e-4297-98e8-b80ef5e8748a kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: Scaled up replica set error-404-isvc-a916f-predictor-5548f55f77 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-a916f-predictor.18c558067612cbff namespace: kserve-ci-e2e-test resourceVersion: "26272" uid: c12260d8-fca0-4f6c-a2ca-ac8a82e7caa1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-24T21:47:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a916f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 2cd4aa48-113f-427e-867c-cf359312828b kind: Event lastTimestamp: "2026-07-24T21:48:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:47:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:07Z" name: error-404-isvc-a916f-predictor.18c55809f47fd813 namespace: kserve-ci-e2e-test resourceVersion: "27205" uid: 4155618b-8a4f-4368-acdc-e0ea6dbf5152 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-24T21:47:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a916f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 2cd4aa48-113f-427e-867c-cf359312828b kind: Event lastTimestamp: "2026-07-24T21:48:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:47:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:07Z" name: error-404-isvc-a916f-predictor.18c55809f4e570f0 namespace: kserve-ci-e2e-test resourceVersion: "27206" uid: 73b4c802-44ab-43a3-ad58-ca8774e11bc1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a916f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26489" uid: 2cd4aa48-113f-427e-867c-cf359312828b kind: Event lastTimestamp: "2026-07-24T21:47:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:22Z" name: error-404-isvc-a916f-predictor.18c5580d7392c4dd namespace: kserve-ci-e2e-test resourceVersion: "26773" uid: adf53198-f755-472e-9209-13b2e9f96568 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a916f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26489" uid: 2cd4aa48-113f-427e-867c-cf359312828b kind: Event lastTimestamp: "2026-07-24T21:47:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:22Z" name: error-404-isvc-a916f-predictor.18c5580d74066504 namespace: kserve-ci-e2e-test resourceVersion: "26775" uid: d87acb97-ea70-4dac-a7bd-18709cbd8c27 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a916f namespace: kserve-ci-e2e-test resourceVersion: "26241" uid: bb8865f3-903f-4ffc-a456-c6df058e7e67 kind: Event lastTimestamp: "2026-07-24T21:47:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:47:40Z" name: error-404-isvc-a916f.18c558067643a3a3 namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: b48a7c61-01cc-4f2c-a115-fd591ec929ba reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a916f namespace: kserve-ci-e2e-test resourceVersion: "26241" uid: bb8865f3-903f-4ffc-a456-c6df058e7e67 kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: 'Failed to update status for InferenceService "error-404-isvc-a916f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a916f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-a916f.18c55806769d0ba0 namespace: kserve-ci-e2e-test resourceVersion: "26276" uid: 2d69f46e-ba58-4b91-8708-6fa6543cea81 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a916f namespace: kserve-ci-e2e-test resourceVersion: "26241" uid: bb8865f3-903f-4ffc-a456-c6df058e7e67 kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a916f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:46:52Z" name: error-404-isvc-a916f.18c55806769d66f0 namespace: kserve-ci-e2e-test resourceVersion: "26279" uid: 1b2437cc-c005-49aa-b3a5-2bd091bd15a2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a916f namespace: kserve-ci-e2e-test resourceVersion: "26926" uid: bb8865f3-903f-4ffc-a456-c6df058e7e67 kind: Event lastTimestamp: "2026-07-24T21:47:40Z" message: InferenceService [error-404-isvc-a916f] is Ready metadata: creationTimestamp: "2026-07-24T21:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:47:40Z" name: error-404-isvc-a916f.18c55811c03b201b namespace: kserve-ci-e2e-test resourceVersion: "26927" uid: aedcdd50-2f86-4b73-8854-1c882f74144e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:35:36.884564Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq namespace: kserve-ci-e2e-test resourceVersion: "20164" uid: ed4d5fa6-13a7-4f14-9442-2e119333e8ae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b6aba-predictor-7b46948747-r5tnq to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:35:36Z" name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq.18c5576940520977 namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: ae75a537-bd6f-47b5-8fb6-1e6648de3215 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: ed4d5fa6-13a7-4f14-9442-2e119333e8ae kind: Event lastTimestamp: "2026-07-24T21:35:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-b6aba-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:37Z" name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq.18c557694a0f5945 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: a4ea1dd4-a0d3-4808-82d4-944872c8eaca reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq namespace: kserve-ci-e2e-test resourceVersion: "20174" uid: ed4d5fa6-13a7-4f14-9442-2e119333e8ae kind: Event lastTimestamp: "2026-07-24T21:35:37Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:35:37Z" name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq.18c557697dd80c0a namespace: kserve-ci-e2e-test resourceVersion: "20204" uid: 15871895-8410-4c69-bb1f-ab55ce4f33ec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: ed4d5fa6-13a7-4f14-9442-2e119333e8ae kind: Event lastTimestamp: "2026-07-24T21:35:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:37Z" name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq.18c557697f2169a5 namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 3371368d-cc60-40ea-92d6-d9a707d90a98 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: ed4d5fa6-13a7-4f14-9442-2e119333e8ae kind: Event lastTimestamp: "2026-07-24T21:35:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:37Z" name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq.18c55769821946b5 namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: 1ff899b8-ea1b-423d-93f1-fc7a8621b077 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: ed4d5fa6-13a7-4f14-9442-2e119333e8ae kind: Event lastTimestamp: "2026-07-24T21:35:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:37Z" name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq.18c55769828cc94a namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: 8ba22f0b-b149-4ef9-86fa-05d86b137b8f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: ed4d5fa6-13a7-4f14-9442-2e119333e8ae kind: Event lastTimestamp: "2026-07-24T21:35:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:38Z" name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq.18c5576982970801 namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: aa35e3b0-162c-40de-97c9-0bd391227616 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: ed4d5fa6-13a7-4f14-9442-2e119333e8ae kind: Event lastTimestamp: "2026-07-24T21:35:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:38Z" name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq.18c557698555db77 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: 97ea9114-7903-46a7-9e8d-053644cd25fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: ed4d5fa6-13a7-4f14-9442-2e119333e8ae kind: Event lastTimestamp: "2026-07-24T21:35:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:38Z" name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq.18c55769860afc13 namespace: kserve-ci-e2e-test resourceVersion: "20212" uid: d8ec47ed-04f2-440f-9562-74f801be533e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:35:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: ed4d5fa6-13a7-4f14-9442-2e119333e8ae kind: Event lastTimestamp: "2026-07-24T21:36:15Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:35:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:15Z" name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq.18c55769cf681b85 namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: 004f8b1b-56e9-4c0d-871b-561434d84ea0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: ed4d5fa6-13a7-4f14-9442-2e119333e8ae kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:47Z" name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq.18c557799feb601f namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: df0d3096-9d9e-4c43-961e-193dab7eeb10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: ed4d5fa6-13a7-4f14-9442-2e119333e8ae kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:47Z" name: error-404-isvc-b6aba-predictor-7b46948747-r5tnq.18c557799febefb2 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 5bdd5039-2c5d-4642-9a44-16fc2f50b315 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b6aba-predictor-7b46948747 namespace: kserve-ci-e2e-test resourceVersion: "20152" uid: 98481589-81d0-42f8-8c2b-7bd8f787bc4f kind: Event lastTimestamp: "2026-07-24T21:35:36Z" message: 'Created pod: error-404-isvc-b6aba-predictor-7b46948747-r5tnq' metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:35:36Z" name: error-404-isvc-b6aba-predictor-7b46948747.18c557693ff26bce namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: 89fcefeb-b55b-4013-93fe-eed04553cc44 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b6aba-predictor namespace: kserve-ci-e2e-test resourceVersion: "20147" uid: 1eee48dd-1207-4570-95c6-646bbc970a9a kind: Event lastTimestamp: "2026-07-24T21:35:36Z" message: Scaled up replica set error-404-isvc-b6aba-predictor-7b46948747 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:35:36Z" name: error-404-isvc-b6aba-predictor.18c557693e50f013 namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: 3e4fc387-04e9-4952-a91b-6e124b84e3ed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:35:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b6aba-predictor namespace: kserve-ci-e2e-test resourceVersion: "20150" uid: 23b583d1-4940-476d-be48-e10a7b84e162 kind: Event lastTimestamp: "2026-07-24T21:36:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:06Z" name: error-404-isvc-b6aba-predictor.18c5576cbcf1de03 namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 3a634066-f350-4fc5-8c92-19496f8c131f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:35:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b6aba-predictor namespace: kserve-ci-e2e-test resourceVersion: "20150" uid: 23b583d1-4940-476d-be48-e10a7b84e162 kind: Event lastTimestamp: "2026-07-24T21:36:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:06Z" name: error-404-isvc-b6aba-predictor.18c5576cbd6fff10 namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: af59842b-fee7-444f-8774-180042d36b7d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:36:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b6aba-predictor namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 23b583d1-4940-476d-be48-e10a7b84e162 kind: Event lastTimestamp: "2026-07-24T21:36:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:36:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:36Z" name: error-404-isvc-b6aba-predictor.18c55773bb0f711a namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 90d2acf5-ac66-423c-85d1-204406348b29 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:36:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b6aba-predictor namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 23b583d1-4940-476d-be48-e10a7b84e162 kind: Event lastTimestamp: "2026-07-24T21:36:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:36:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:36Z" name: error-404-isvc-b6aba-predictor.18c55773bb84596d namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: ee7dd0af-ade1-44d6-9b1d-23c13528d089 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b6aba namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: c4e28392-3d41-4797-b394-d0b721d59f63 kind: Event lastTimestamp: "2026-07-24T21:36:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:36:25Z" name: error-404-isvc-b6aba.18c557693e454935 namespace: kserve-ci-e2e-test resourceVersion: "20605" uid: 7feb7873-27e5-40dd-999b-bfec9d8911f9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b6aba namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: c4e28392-3d41-4797-b394-d0b721d59f63 kind: Event lastTimestamp: "2026-07-24T21:35:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-b6aba": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b6aba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:35:36Z" name: error-404-isvc-b6aba.18c557693ed4b15e namespace: kserve-ci-e2e-test resourceVersion: "20160" uid: f0e74759-c7af-4593-bd80-7571b7c2eb2a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b6aba namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: c4e28392-3d41-4797-b394-d0b721d59f63 kind: Event lastTimestamp: "2026-07-24T21:35:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b6aba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:35:36Z" name: error-404-isvc-b6aba.18c557693ed50eee namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: e5acf4cc-bb6b-43ba-a9b2-65aea61b3252 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b6aba namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: c4e28392-3d41-4797-b394-d0b721d59f63 kind: Event lastTimestamp: "2026-07-24T21:36:25Z" message: InferenceService [error-404-isvc-b6aba] is Ready metadata: creationTimestamp: "2026-07-24T21:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:36:25Z" name: error-404-isvc-b6aba.18c557748c683fe1 namespace: kserve-ci-e2e-test resourceVersion: "20604" uid: 6f44a1b6-e919-40ec-816d-54370db704f1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:47:12.556032Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: 72e26efb-19f4-4911-95e3-91a0705615cd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:47:12Z" name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b.18c5580b3992370a namespace: kserve-ci-e2e-test resourceVersion: "26642" uid: f29aeda6-5945-43b4-9b44-92894967af6b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b namespace: kserve-ci-e2e-test resourceVersion: "26650" uid: 72e26efb-19f4-4911-95e3-91a0705615cd kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: Add eth0 [10.132.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:47:12Z" name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b.18c5580b53441e83 namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: 923e65a3-8614-40ab-9725-260d8fa68245 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 72e26efb-19f4-4911-95e3-91a0705615cd kind: Event lastTimestamp: "2026-07-24T21:47:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:13Z" name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b.18c5580b54839c5b namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: 9d8b9415-5c03-4a39-829c-4bce4333e306 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 72e26efb-19f4-4911-95e3-91a0705615cd kind: Event lastTimestamp: "2026-07-24T21:47:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:13Z" name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b.18c5580b57931359 namespace: kserve-ci-e2e-test resourceVersion: "26671" uid: fc16f40b-85f5-4afe-bc01-9d4123635ae3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 72e26efb-19f4-4911-95e3-91a0705615cd kind: Event lastTimestamp: "2026-07-24T21:47:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:13Z" name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b.18c5580b5804d9de namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: 06ed5295-bb70-4ffa-8189-efc9e0ca4486 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 72e26efb-19f4-4911-95e3-91a0705615cd kind: Event lastTimestamp: "2026-07-24T21:47:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:13Z" name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b.18c5580b580db049 namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: b0d1a311-24d9-4290-bea0-e8a8e6123db7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 72e26efb-19f4-4911-95e3-91a0705615cd kind: Event lastTimestamp: "2026-07-24T21:47:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:13Z" name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b.18c5580b5b73dcc9 namespace: kserve-ci-e2e-test resourceVersion: "26674" uid: 7d949f7d-26d6-4073-89ea-e8204acd90ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 72e26efb-19f4-4911-95e3-91a0705615cd kind: Event lastTimestamp: "2026-07-24T21:47:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:13Z" name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b.18c5580b5be933d1 namespace: kserve-ci-e2e-test resourceVersion: "26675" uid: b1830dea-5257-4953-a73c-a0e2d86e7ad6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-24T21:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 72e26efb-19f4-4911-95e3-91a0705615cd kind: Event lastTimestamp: "2026-07-24T21:56:29Z" message: 'Readiness probe failed: dial tcp 10.132.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:56:29Z" name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b.18c5580b772de725 namespace: kserve-ci-e2e-test resourceVersion: "30779" uid: 597b46bf-3bc3-4ede-8340-7e11e69f510a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:56:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 72e26efb-19f4-4911-95e3-91a0705615cd kind: Event lastTimestamp: "2026-07-24T21:56:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:56:27Z" name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b.18c5588c66792516 namespace: kserve-ci-e2e-test resourceVersion: "30758" uid: 958e4319-5b1a-4986-b522-e0c519e236f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:56:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 72e26efb-19f4-4911-95e3-91a0705615cd kind: Event lastTimestamp: "2026-07-24T21:56:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:56:27Z" name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b.18c5588c6679da87 namespace: kserve-ci-e2e-test resourceVersion: "30759" uid: bd9c49d7-e5aa-423e-8ae0-085c9f386468 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:56:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 72e26efb-19f4-4911-95e3-91a0705615cd kind: Event lastTimestamp: "2026-07-24T21:56:29Z" message: 'Readiness probe failed: Get "https://10.132.0.48:8643/healthz": dial tcp 10.132.0.48:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:56:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:56:29Z" name: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b.18c5588ceb9bde88 namespace: kserve-ci-e2e-test resourceVersion: "30778" uid: a8c79694-b586-487e-a9a4-81cc21a0085d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bce0e-predictor-8fffdbfb9 namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: f7667417-3cc4-4f31-9d18-b456bd033403 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: 'Created pod: error-404-isvc-bce0e-predictor-8fffdbfb9-78z7b' metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:12Z" name: error-404-isvc-bce0e-predictor-8fffdbfb9.18c5580b39229f19 namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: bb8e2fcf-403d-4e9b-bec4-834c1fd143fb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bce0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: 7aebeaa3-c531-4c61-9659-c1dfa4dc5689 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: Scaled up replica set error-404-isvc-bce0e-predictor-8fffdbfb9 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:12Z" name: error-404-isvc-bce0e-predictor.18c5580b37b05167 namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: 6b41f3f9-a9ab-49f1-86c2-55e80890950a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:47:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bce0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26630" uid: 9a737aa0-b011-4b14-b86f-c99b4ed8f7a6 kind: Event lastTimestamp: "2026-07-24T21:47:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:42Z" name: error-404-isvc-bce0e-predictor.18c5580eb6f5fe42 namespace: kserve-ci-e2e-test resourceVersion: "26954" uid: 833dd58a-e087-41e1-9293-93c7ca83ed36 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:47:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bce0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26630" uid: 9a737aa0-b011-4b14-b86f-c99b4ed8f7a6 kind: Event lastTimestamp: "2026-07-24T21:47:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:42Z" name: error-404-isvc-bce0e-predictor.18c5580eb7ef8d76 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 1e055b8c-693d-4998-b848-801a9cf9359b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:47:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bce0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26818" uid: 9a737aa0-b011-4b14-b86f-c99b4ed8f7a6 kind: Event lastTimestamp: "2026-07-24T21:48:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:47:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:12Z" name: error-404-isvc-bce0e-predictor.18c55815b5598502 namespace: kserve-ci-e2e-test resourceVersion: "27271" uid: cf26f36e-0a45-4d3e-8c53-fc670feabc18 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:47:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bce0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26818" uid: 9a737aa0-b011-4b14-b86f-c99b4ed8f7a6 kind: Event lastTimestamp: "2026-07-24T21:48:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:47:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:12Z" name: error-404-isvc-bce0e-predictor.18c55815b5caae34 namespace: kserve-ci-e2e-test resourceVersion: "27272" uid: aa953679-c676-4bbb-9e77-6ae777107816 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bce0e namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 5980d333-2128-449c-9f0f-db3aefd13738 kind: Event lastTimestamp: "2026-07-24T21:47:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:47:59Z" name: error-404-isvc-bce0e.18c5580b37b2902a namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: e31c3169-ce75-4e2b-b4bc-2d37bd8fcf29 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bce0e namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 5980d333-2128-449c-9f0f-db3aefd13738 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-bce0e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bce0e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:47:12Z" name: error-404-isvc-bce0e.18c5580b383e98bc namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: 2758fd4f-90b8-475d-888e-9ed70ff41bf0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bce0e namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 5980d333-2128-449c-9f0f-db3aefd13738 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bce0e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:47:12Z" name: error-404-isvc-bce0e.18c5580b383ee3d6 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: c8f0c434-221b-434c-9b9c-081408721e36 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bce0e namespace: kserve-ci-e2e-test resourceVersion: "27130" uid: 5980d333-2128-449c-9f0f-db3aefd13738 kind: Event lastTimestamp: "2026-07-24T21:47:59Z" message: InferenceService [error-404-isvc-bce0e] is Ready metadata: creationTimestamp: "2026-07-24T21:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:47:59Z" name: error-404-isvc-bce0e.18c5581635d297e6 namespace: kserve-ci-e2e-test resourceVersion: "27131" uid: d12e8adc-6cb8-4938-9c3d-948115cf4cda reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:27:11.945540Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk namespace: kserve-ci-e2e-test resourceVersion: "16402" uid: 9dee7a60-efe2-4921-bd3d-dd373fb7b197 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c3da7-predictor-6d65f47599-b48qk to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:27:11Z" name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk.18c556f3af9bfd74 namespace: kserve-ci-e2e-test resourceVersion: "16407" uid: efb39d02-80e8-45c2-aed6-87756ef9e42b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 9dee7a60-efe2-4921-bd3d-dd373fb7b197 kind: Event lastTimestamp: "2026-07-24T21:27:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c3da7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:12Z" name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk.18c556f3bdb73b60 namespace: kserve-ci-e2e-test resourceVersion: "16424" uid: 0d991199-d7b8-4172-be07-1923fbc6cbc4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: 9dee7a60-efe2-4921-bd3d-dd373fb7b197 kind: Event lastTimestamp: "2026-07-24T21:27:12Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:27:12Z" name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk.18c556f3ed01c1f6 namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: 18f8316a-ce7a-43e2-9771-1b7d45a1da92 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 9dee7a60-efe2-4921-bd3d-dd373fb7b197 kind: Event lastTimestamp: "2026-07-24T21:27:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:12Z" name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk.18c556f3ee4659bc namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: c5693940-a157-4cc2-93a6-287462e9cdb4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 9dee7a60-efe2-4921-bd3d-dd373fb7b197 kind: Event lastTimestamp: "2026-07-24T21:27:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:13Z" name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk.18c556f3f1249710 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: e41f6a76-6049-41e5-bbb5-d0ad131aea63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 9dee7a60-efe2-4921-bd3d-dd373fb7b197 kind: Event lastTimestamp: "2026-07-24T21:27:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:13Z" name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk.18c556f3f1ac139d namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: d9a76909-0041-4290-bcd4-4da8ce5d71a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 9dee7a60-efe2-4921-bd3d-dd373fb7b197 kind: Event lastTimestamp: "2026-07-24T21:27:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:13Z" name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk.18c556f3f1b4a78f namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: ee89a197-b338-4c5c-8017-154ec8c3dc46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 9dee7a60-efe2-4921-bd3d-dd373fb7b197 kind: Event lastTimestamp: "2026-07-24T21:27:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:13Z" name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk.18c556f3f4356579 namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: f3aa3815-a874-4d12-add8-21b14c19beab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 9dee7a60-efe2-4921-bd3d-dd373fb7b197 kind: Event lastTimestamp: "2026-07-24T21:27:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:13Z" name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk.18c556f3f4a6376a namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 3db2a61f-4473-46f1-9b4a-432ac75c06f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 9dee7a60-efe2-4921-bd3d-dd373fb7b197 kind: Event lastTimestamp: "2026-07-24T21:27:49Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:49Z" name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk.18c556f424f864c6 namespace: kserve-ci-e2e-test resourceVersion: "16806" uid: 13307f0b-f914-4029-96c4-cb966021c97b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 9dee7a60-efe2-4921-bd3d-dd373fb7b197 kind: Event lastTimestamp: "2026-07-24T21:36:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:26Z" name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk.18c55774dcc812fa namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: 52f74546-86b6-4da0-9ac4-7488d54ed22a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 9dee7a60-efe2-4921-bd3d-dd373fb7b197 kind: Event lastTimestamp: "2026-07-24T21:36:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:26Z" name: error-404-isvc-c3da7-predictor-6d65f47599-b48qk.18c55774dcc87ef2 namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 7b6b4ebf-7f96-44c6-b07c-688afea5a31f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c3da7-predictor-6d65f47599 namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 55f40b68-6e57-45e3-80c5-a26048dd5b8e kind: Event lastTimestamp: "2026-07-24T21:27:11Z" message: 'Created pod: error-404-isvc-c3da7-predictor-6d65f47599-b48qk' metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:11Z" name: error-404-isvc-c3da7-predictor-6d65f47599.18c556f3af2cf91d namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: 84c40c80-3ade-484d-aba6-67f1cb594c55 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c3da7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16389" uid: 1e077520-e8eb-410d-b1af-93f0675ff0c8 kind: Event lastTimestamp: "2026-07-24T21:27:11Z" message: Scaled up replica set error-404-isvc-c3da7-predictor-6d65f47599 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:11Z" name: error-404-isvc-c3da7-predictor.18c556f3ad565b7d namespace: kserve-ci-e2e-test resourceVersion: "16395" uid: b3b4df00-6f9c-4e51-a740-05928d98bb66 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3da7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: e58edbd4-6613-4359-9da2-07aa76035bc6 kind: Event lastTimestamp: "2026-07-24T21:27:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:26Z" name: error-404-isvc-c3da7-predictor.18c556f72bfdd947 namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: 94f700e2-eb1c-4d94-92c5-0f6fe8a43694 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3da7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: e58edbd4-6613-4359-9da2-07aa76035bc6 kind: Event lastTimestamp: "2026-07-24T21:27:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:26Z" name: error-404-isvc-c3da7-predictor.18c556f72c69c0ab namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: bc85782e-05e2-45bd-bb1f-7059f646a27d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-24T21:27:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3da7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: e58edbd4-6613-4359-9da2-07aa76035bc6 kind: Event lastTimestamp: "2026-07-24T21:28:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:28:26Z" name: error-404-isvc-c3da7-predictor.18c556faab27884c namespace: kserve-ci-e2e-test resourceVersion: "17138" uid: 2d333559-e0cb-43b2-b2e4-1ca606f67127 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-24T21:27:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3da7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: e58edbd4-6613-4359-9da2-07aa76035bc6 kind: Event lastTimestamp: "2026-07-24T21:28:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:28:26Z" name: error-404-isvc-c3da7-predictor.18c556faab91474b namespace: kserve-ci-e2e-test resourceVersion: "17139" uid: 7939a866-ce4a-4e39-b09f-56dff90882f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3da7 namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: 357abd04-afa1-47ed-b917-a5416b253afc kind: Event lastTimestamp: "2026-07-24T21:28:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:28:00Z" name: error-404-isvc-c3da7.18c556f3ad9a720a namespace: kserve-ci-e2e-test resourceVersion: "16899" uid: 41f079a0-b794-46f8-b6e2-048126b1d0c9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3da7 namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: 357abd04-afa1-47ed-b917-a5416b253afc kind: Event lastTimestamp: "2026-07-24T21:27:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-c3da7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3da7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:27:11Z" name: error-404-isvc-c3da7.18c556f3adf9c007 namespace: kserve-ci-e2e-test resourceVersion: "16399" uid: 3bf53ea9-f408-431f-bc8f-c96f8a0a1ba3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3da7 namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: 357abd04-afa1-47ed-b917-a5416b253afc kind: Event lastTimestamp: "2026-07-24T21:27:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3da7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:27:11Z" name: error-404-isvc-c3da7.18c556f3adf9f117 namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: d531639c-5350-472a-9d25-22958d4301de reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:28:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3da7 namespace: kserve-ci-e2e-test resourceVersion: "16897" uid: 357abd04-afa1-47ed-b917-a5416b253afc kind: Event lastTimestamp: "2026-07-24T21:28:00Z" message: InferenceService [error-404-isvc-c3da7] is Ready metadata: creationTimestamp: "2026-07-24T21:28:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:28:00Z" name: error-404-isvc-c3da7.18c556fee008f8e5 namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: 99a778fc-b9cf-4895-b0f8-36e3a931c274 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:26:21.992420Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: c798a1f6-74e8-48e5-ae62-fc1a546d596a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-eea53-predictor-6bd495b56f-7z42x to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:26:21Z" name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x.18c556e80e2c1add namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: b5b56a67-dcb1-4e17-9eb2-c3204bb9ad5b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: c798a1f6-74e8-48e5-ae62-fc1a546d596a kind: Event lastTimestamp: "2026-07-24T21:26:22Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:26:22Z" name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x.18c556e827e9809c namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: 6ade2700-2399-4d7c-ad1e-0051fc8df0fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: c798a1f6-74e8-48e5-ae62-fc1a546d596a kind: Event lastTimestamp: "2026-07-24T21:26:22Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:22Z" name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x.18c556e8294b7c1b namespace: kserve-ci-e2e-test resourceVersion: "15789" uid: adb7605a-faad-443e-a5ce-2b5e2f9b45fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: c798a1f6-74e8-48e5-ae62-fc1a546d596a kind: Event lastTimestamp: "2026-07-24T21:26:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:22Z" name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x.18c556e82c24853e namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: a0fddf9a-8e85-4272-a89c-943e8679a5e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: c798a1f6-74e8-48e5-ae62-fc1a546d596a kind: Event lastTimestamp: "2026-07-24T21:26:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:22Z" name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x.18c556e82c990332 namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: 9de0d641-61ee-4c5a-a4c6-c3f80f636a61 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: c798a1f6-74e8-48e5-ae62-fc1a546d596a kind: Event lastTimestamp: "2026-07-24T21:26:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:22Z" name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x.18c556e82ca21c97 namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: bf868f52-7e9d-4bc3-bead-93ac5f8925a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: c798a1f6-74e8-48e5-ae62-fc1a546d596a kind: Event lastTimestamp: "2026-07-24T21:26:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:22Z" name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x.18c556e82f32ed77 namespace: kserve-ci-e2e-test resourceVersion: "15793" uid: d9eabbf8-fd61-42cf-92c7-7107725ff43f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: c798a1f6-74e8-48e5-ae62-fc1a546d596a kind: Event lastTimestamp: "2026-07-24T21:26:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:22Z" name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x.18c556e82fa68041 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 2214bc12-f998-4688-9d7f-84b0f874345e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-24T21:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: c798a1f6-74e8-48e5-ae62-fc1a546d596a kind: Event lastTimestamp: "2026-07-24T21:35:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:38Z" name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x.18c556e836d6cd62 namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 1e5b1d47-8619-4e75-9a41-f116637d13c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: c798a1f6-74e8-48e5-ae62-fc1a546d596a kind: Event lastTimestamp: "2026-07-24T21:35:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:36Z" name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x.18c557693e73026f namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: 1236dec2-9868-4432-a281-03340a080404 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: c798a1f6-74e8-48e5-ae62-fc1a546d596a kind: Event lastTimestamp: "2026-07-24T21:35:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:36Z" name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x.18c557693e738951 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: 78dd73bc-a137-439a-9a50-9a7aa5773a12 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: c798a1f6-74e8-48e5-ae62-fc1a546d596a kind: Event lastTimestamp: "2026-07-24T21:35:38Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": dial tcp 10.132.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:38Z" name: error-404-isvc-eea53-predictor-6bd495b56f-7z42x.18c55769ab5299de namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: f2737e76-33c4-493e-8e0d-c7e0f8806c3a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-eea53-predictor-6bd495b56f namespace: kserve-ci-e2e-test resourceVersion: "15750" uid: 2091824f-81e5-4b32-a76d-ee21b109a1ec kind: Event lastTimestamp: "2026-07-24T21:26:21Z" message: 'Created pod: error-404-isvc-eea53-predictor-6bd495b56f-7z42x' metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:21Z" name: error-404-isvc-eea53-predictor-6bd495b56f.18c556e80dc420f6 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: a10fe6ff-5646-4bcb-913d-26fe0f8f774d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-eea53-predictor namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 36b06350-fcb3-484a-82a1-2f69eaf2f2da kind: Event lastTimestamp: "2026-07-24T21:26:21Z" message: Scaled up replica set error-404-isvc-eea53-predictor-6bd495b56f from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:21Z" name: error-404-isvc-eea53-predictor.18c556e80bfa127e namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: feb2dc64-bf61-41f2-96e3-2a1b46decc1f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:26:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eea53-predictor namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: 2c8bc3be-0a21-4172-a756-a326c6e395be kind: Event lastTimestamp: "2026-07-24T21:26:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:51Z" name: error-404-isvc-eea53-predictor.18c556eb8adc2b82 namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: 23f1b390-69b2-4b10-89c6-62f2249143df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:26:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eea53-predictor namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: 2c8bc3be-0a21-4172-a756-a326c6e395be kind: Event lastTimestamp: "2026-07-24T21:26:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:51Z" name: error-404-isvc-eea53-predictor.18c556eb8b466443 namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: afe2862e-933d-4e7a-a44b-0241ed19b983 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:27:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eea53-predictor namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 2c8bc3be-0a21-4172-a756-a326c6e395be kind: Event lastTimestamp: "2026-07-24T21:27:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:27:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:37Z" name: error-404-isvc-eea53-predictor.18c556f2887dff87 namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: 43a25928-0f67-4ec6-b793-401a91f7b1e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:27:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eea53-predictor namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 2c8bc3be-0a21-4172-a756-a326c6e395be kind: Event lastTimestamp: "2026-07-24T21:27:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:27:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:37Z" name: error-404-isvc-eea53-predictor.18c556f288e54729 namespace: kserve-ci-e2e-test resourceVersion: "16702" uid: ff4ac7eb-9fa6-439e-bee4-934593a4e86c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:26:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-eea53 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: 01566fce-6fe7-4724-af4f-dd24b436437e kind: Event lastTimestamp: "2026-07-24T21:27:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:27:08Z" name: error-404-isvc-eea53.18c556e80bfc1bda namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: c56ecad2-cb7e-4b9a-8600-6e92482e2e6b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-eea53 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: 01566fce-6fe7-4724-af4f-dd24b436437e kind: Event lastTimestamp: "2026-07-24T21:26:21Z" message: 'Failed to update status for InferenceService "error-404-isvc-eea53": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-eea53": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:26:21Z" name: error-404-isvc-eea53.18c556e80c91e3e7 namespace: kserve-ci-e2e-test resourceVersion: "15755" uid: 9c712421-b120-4208-b4a6-632260fa91e5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-eea53 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: 01566fce-6fe7-4724-af4f-dd24b436437e kind: Event lastTimestamp: "2026-07-24T21:26:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-eea53": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:26:21Z" name: error-404-isvc-eea53.18c556e80c9229d0 namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: 12ee893a-9cfe-42bc-b6bf-9e6b79c77938 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-eea53 namespace: kserve-ci-e2e-test resourceVersion: "16253" uid: 01566fce-6fe7-4724-af4f-dd24b436437e kind: Event lastTimestamp: "2026-07-24T21:27:08Z" message: InferenceService [error-404-isvc-eea53] is Ready metadata: creationTimestamp: "2026-07-24T21:27:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:27:08Z" name: error-404-isvc-eea53.18c556f2f1c136f6 namespace: kserve-ci-e2e-test resourceVersion: "16255" uid: 1e5f6648-3a68-428e-86fc-4e1204071025 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:36:27.024704Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l namespace: kserve-ci-e2e-test resourceVersion: "20711" uid: b5fe3572-3832-4f45-bd9e-7124da1903cc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f761f-predictor-54645fb6f5-cf88l to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:36:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:36:27Z" name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l.18c55774ece7e7ef namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: 63441fcd-a5ee-441b-a813-3398b7727989 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: b5fe3572-3832-4f45-bd9e-7124da1903cc kind: Event lastTimestamp: "2026-07-24T21:36:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f761f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:27Z" name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l.18c55774f87c865b namespace: kserve-ci-e2e-test resourceVersion: "20731" uid: 1a75b3b8-6789-4b8f-938d-d6e0338130ca reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: b5fe3572-3832-4f45-bd9e-7124da1903cc kind: Event lastTimestamp: "2026-07-24T21:36:28Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:36:28Z" name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l.18c557752a4f98ad namespace: kserve-ci-e2e-test resourceVersion: "20748" uid: 4fdaa403-ec0e-430f-b69a-1d2689c854d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: b5fe3572-3832-4f45-bd9e-7124da1903cc kind: Event lastTimestamp: "2026-07-24T21:36:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:28Z" name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l.18c557752c0c4438 namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: 535ef71d-b1ff-45a0-8ce6-255b0305e6be reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: b5fe3572-3832-4f45-bd9e-7124da1903cc kind: Event lastTimestamp: "2026-07-24T21:36:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:28Z" name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l.18c557752efbdf58 namespace: kserve-ci-e2e-test resourceVersion: "20754" uid: 5ef1009f-e5d1-4eed-9d1a-97635f1e4ff6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: b5fe3572-3832-4f45-bd9e-7124da1903cc kind: Event lastTimestamp: "2026-07-24T21:36:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:28Z" name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l.18c557752f6c6bb9 namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 1d148dc7-7b6e-45bd-a878-db50dabe9972 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: b5fe3572-3832-4f45-bd9e-7124da1903cc kind: Event lastTimestamp: "2026-07-24T21:36:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:28Z" name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l.18c557752f7513cb namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: b504f4ad-a1a9-412e-b490-14131b1e2131 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: b5fe3572-3832-4f45-bd9e-7124da1903cc kind: Event lastTimestamp: "2026-07-24T21:36:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:28Z" name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l.18c5577532f0ff86 namespace: kserve-ci-e2e-test resourceVersion: "20757" uid: 3c357597-9c38-4836-9fdc-cf60a24d5cc4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: b5fe3572-3832-4f45-bd9e-7124da1903cc kind: Event lastTimestamp: "2026-07-24T21:36:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:28Z" name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l.18c557753363f8a2 namespace: kserve-ci-e2e-test resourceVersion: "20758" uid: fbe9ab9c-6443-4555-8f33-a7861b34d8eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:36:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: b5fe3572-3832-4f45-bd9e-7124da1903cc kind: Event lastTimestamp: "2026-07-24T21:37:05Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:05Z" name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l.18c55775832fd59e namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 6466ba69-786a-4b42-97c6-d9bd47d1301c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: b5fe3572-3832-4f45-bd9e-7124da1903cc kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l.18c557853cc398cb namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: 16502869-3f56-458b-8e49-896f93b97ece reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: b5fe3572-3832-4f45-bd9e-7124da1903cc kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: error-404-isvc-f761f-predictor-54645fb6f5-cf88l.18c557853cc416be namespace: kserve-ci-e2e-test resourceVersion: "21650" uid: 111b8dce-1d08-4435-80b5-dd4d7cf3f92e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f761f-predictor-54645fb6f5 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 255c1dbe-ad28-4a7b-8a5b-1569520502e2 kind: Event lastTimestamp: "2026-07-24T21:36:27Z" message: 'Created pod: error-404-isvc-f761f-predictor-54645fb6f5-cf88l' metadata: creationTimestamp: "2026-07-24T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:27Z" name: error-404-isvc-f761f-predictor-54645fb6f5.18c55774ec039b4e namespace: kserve-ci-e2e-test resourceVersion: "20713" uid: 5b8d1e4c-e377-4153-b867-21739bcdf24e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f761f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: 1dc97383-abe7-4f9f-9bd9-c709a4339b80 kind: Event lastTimestamp: "2026-07-24T21:36:26Z" message: Scaled up replica set error-404-isvc-f761f-predictor-54645fb6f5 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:26Z" name: error-404-isvc-f761f-predictor.18c55774e82888bb namespace: kserve-ci-e2e-test resourceVersion: "20705" uid: 5a72faf1-28d9-4a8c-a618-6d6b9a229c1a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:36:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f761f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20702" uid: 465efba1-1f1b-4d80-a547-667c3cc134d7 kind: Event lastTimestamp: "2026-07-24T21:37:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:11Z" name: error-404-isvc-f761f-predictor.18c55778661d6ff2 namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: efdc75a3-68d1-4b26-a441-eae0e871e9df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:36:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f761f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20702" uid: 465efba1-1f1b-4d80-a547-667c3cc134d7 kind: Event lastTimestamp: "2026-07-24T21:37:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:11Z" name: error-404-isvc-f761f-predictor.18c5577866894202 namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: dfa94b41-2450-4a69-bd92-727921416f19 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f761f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 465efba1-1f1b-4d80-a547-667c3cc134d7 kind: Event lastTimestamp: "2026-07-24T21:37:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:26Z" name: error-404-isvc-f761f-predictor.18c55782e2fe1b47 namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: f26cfa5f-5071-4fec-9bae-7a081090dcf8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f761f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 465efba1-1f1b-4d80-a547-667c3cc134d7 kind: Event lastTimestamp: "2026-07-24T21:37:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:26Z" name: error-404-isvc-f761f-predictor.18c55782e35e5827 namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: bbe82391-7bf6-43b1-9166-58e26aa431f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f761f namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 40d3077e-4439-4821-a4eb-b8e931a075c2 kind: Event lastTimestamp: "2026-07-24T21:37:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:15Z" name: error-404-isvc-f761f.18c55774e8273af2 namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: f6180dc3-d528-47d7-81a6-f48384e4b093 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f761f namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 40d3077e-4439-4821-a4eb-b8e931a075c2 kind: Event lastTimestamp: "2026-07-24T21:36:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-f761f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f761f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:36:26Z" name: error-404-isvc-f761f.18c55774e8fff481 namespace: kserve-ci-e2e-test resourceVersion: "20707" uid: 6d9e01d9-9e58-46af-9e5a-c17e74f219c9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f761f namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 40d3077e-4439-4821-a4eb-b8e931a075c2 kind: Event lastTimestamp: "2026-07-24T21:36:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f761f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:36:26Z" name: error-404-isvc-f761f.18c55774e900311c namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: d9494b7c-75fa-4128-8378-ec87c9ffd695 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f761f namespace: kserve-ci-e2e-test resourceVersion: "21360" uid: 40d3077e-4439-4821-a4eb-b8e931a075c2 kind: Event lastTimestamp: "2026-07-24T21:37:15Z" message: InferenceService [error-404-isvc-f761f] is Ready metadata: creationTimestamp: "2026-07-24T21:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:15Z" name: error-404-isvc-f761f.18c557803e8e19c7 namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: feb865b4-7f95-4437-9bd2-d0fa28f90785 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:24:51.815547Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:24:51Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d30f360d57 namespace: kserve-ci-e2e-test resourceVersion: "14692" uid: 9ecccd14-066a-4fda-9eb1-bf311bfc09a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:24:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:52Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d3175cd319 namespace: kserve-ci-e2e-test resourceVersion: "14793" uid: b0f7009e-2cf4-476e-ae9d-aefba949df89 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:53Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14699" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:24:53Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:24:53Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d3891a22e5 namespace: kserve-ci-e2e-test resourceVersion: "14831" uid: 6314e6f0-8ff1-4d94-95b3-6f8a24fb85b5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:24:53Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4bc99d27049df235a3e7474685f2095e74c6eed89dc3da2f17a1c3e85c3fa894" already present on machine metadata: creationTimestamp: "2026-07-24T21:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:53Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d38ae18e63 namespace: kserve-ci-e2e-test resourceVersion: "14833" uid: 9bff65fa-d87c-4e6f-9c16-2c62558ddb8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:24:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-24T21:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:53Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d38f8af976 namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: 3a9702fb-1876-4fd3-a5fb-6d7708896d73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:24:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-24T21:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:53Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d39137e4cd namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: 4a56c858-b941-4f1c-bff9-19a50a7c9173 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:25:01Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1716" metadata: creationTimestamp: "2026-07-24T21:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:01Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d55905cc08 namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 81ed244e-d693-4249-9beb-042045391477 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:25:09Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1716" in 7.881s (7.881s including waiting). Image size: 1557885864 bytes.' metadata: creationTimestamp: "2026-07-24T21:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:09Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d72ecc9215 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 22ef29f4-daa6-4df6-b685-d8fbfa1d9746 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:25:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:09Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d73546b8da namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: 31184518-164e-4d4b-88b2-d62014848020 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:25:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:09Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d736781f3f namespace: kserve-ci-e2e-test resourceVersion: "15001" uid: d233c4bb-1077-44f3-9ad3-a8dfb18f2972 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:25:09Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-24T21:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:09Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d73686f784 namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: 58309fc9-5103-4ff3-bc96-63128e7428d4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:25:14Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.389s (4.389s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-24T21:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:14Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d83c3147f3 namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: 8ea0b17a-4303-4f18-a96e-a0e309fa9e3b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:25:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:14Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d84165b28c namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: 7aa36f7e-1638-467a-bc50-60d1ac12199f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:25:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:14Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d84343f6a8 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 9db13701-b261-4b33-8f6a-db36f9f5245b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-24T21:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:27:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:15Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556d865f93885 namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 6f5df8e8-73a7-4789-8d59-c2a70332f7d9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:27:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:11Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556f3a7db6dc3 namespace: kserve-ci-e2e-test resourceVersion: "16355" uid: c0ceb555-3d2a-48db-8e46-ccea8e6741ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:27:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:11Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556f3a7dbea0c namespace: kserve-ci-e2e-test resourceVersion: "16358" uid: a6111922-4e96-46a8-ad7a-8a38d23bd8c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: d5bab9a9-4aa2-41e0-8aa4-34eddd2e5abb kind: Event lastTimestamp: "2026-07-24T21:27:15Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:15Z" name: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72.18c556f4928ebcc5 namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 942a2a6d-9b87-4a6f-a243-1c231bd1432d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-7656b4965b namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: b989d645-ab12-4d08-9fcf-97e2fc111b95 kind: Event lastTimestamp: "2026-07-24T21:24:51Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-7656b4965b-sbh72' metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:24:51Z" name: isvc-sklearn-graph-1-predictor-7656b4965b.18c556d30ec9e54c namespace: kserve-ci-e2e-test resourceVersion: "14689" uid: 2e307764-9c01-4a3d-9195-703f2a217474 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: 533408d5-52e5-4188-9507-64babda60642 kind: Event lastTimestamp: "2026-07-24T21:24:51Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-7656b4965b from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:24:51Z" name: isvc-sklearn-graph-1-predictor.18c556d30b0d6d24 namespace: kserve-ci-e2e-test resourceVersion: "14682" uid: 4103d857-bbea-418e-a658-72e678e59d00 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:25:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: f1ab747e-534b-44d2-816b-90fb286cc12c kind: Event lastTimestamp: "2026-07-24T21:25:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:25:21Z" name: isvc-sklearn-graph-1-predictor.18c556d6899c546e namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: 961cde20-68fd-4548-bc93-c74174e49ac4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:25:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: f1ab747e-534b-44d2-816b-90fb286cc12c kind: Event lastTimestamp: "2026-07-24T21:25:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:25:21Z" name: isvc-sklearn-graph-1-predictor.18c556d68a0764f4 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: b447ca81-9fc9-43a8-afc5-6120af86c8e3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:25:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: f1ab747e-534b-44d2-816b-90fb286cc12c kind: Event lastTimestamp: "2026-07-24T21:26:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:51Z" name: isvc-sklearn-graph-1-predictor.18c556dd8763c824 namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: db4d114c-dc68-430f-9a01-2eac33cc9536 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:25:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: f1ab747e-534b-44d2-816b-90fb286cc12c kind: Event lastTimestamp: "2026-07-24T21:26:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:51Z" name: isvc-sklearn-graph-1-predictor.18c556dd87df8911 namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: b9a7e35e-8372-4beb-9974-6db7dfd74bc3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: 7b5dab21-2447-4bdb-be7b-a5a5872e561a kind: Event lastTimestamp: "2026-07-24T21:26:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:26:36Z" name: isvc-sklearn-graph-1.18c556d30b3667f2 namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 11753074-fa68-427d-b914-143a849fdc4c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15950" uid: 7b5dab21-2447-4bdb-be7b-a5a5872e561a kind: Event lastTimestamp: "2026-07-24T21:26:36Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-24T21:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:26:36Z" name: isvc-sklearn-graph-1.18c556eb535bcda0 namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: 48faf48a-0bff-4474-b799-aee0956834c3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:24:52.230420Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14770" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-96447656d-6bksn to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-07-24T21:24:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:24:52Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d327f073b7 namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: 63b0ecd2-8958-4c87-b583-d610aedf7aaf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:24:52Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:24:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:24:52Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d3457b362a namespace: kserve-ci-e2e-test resourceVersion: "14804" uid: f1d0e8bd-ed6c-41ee-a2a1-703427eedfb6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:24:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4bc99d27049df235a3e7474685f2095e74c6eed89dc3da2f17a1c3e85c3fa894" already present on machine metadata: creationTimestamp: "2026-07-24T21:24:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:52Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d347aa90f3 namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: 29e6d3a0-ad6e-445a-8717-eb0604047fbb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:24:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-24T21:24:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:52Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d34c880487 namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: b92f6a68-9d8c-44da-85c7-4f5b4c703aff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:24:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-24T21:24:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:52Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d34d753d45 namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 7b92060d-3b38-4d88-9baf-222de8c133d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:24:58Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1716" metadata: creationTimestamp: "2026-07-24T21:24:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:58Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d4a4bff8ee namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: 688ef7b6-d7da-48bb-8367-e7999783063c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:25:08Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1716" in 9.937s (9.937s including waiting). Image size: 1557885864 bytes.' metadata: creationTimestamp: "2026-07-24T21:25:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:08Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d6f51515ef namespace: kserve-ci-e2e-test resourceVersion: "14983" uid: 079b4d0f-874b-40e6-9913-3514c4a7ad58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:25:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:25:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:08Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d6f8cfe206 namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: 7fe450a4-21b0-4162-a482-2d14353470b1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:25:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:25:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:08Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d6f94d4a3c namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 9059a13b-9861-4aed-8e45-65258964c874 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:25:08Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-24T21:25:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:08Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d6f95c90f7 namespace: kserve-ci-e2e-test resourceVersion: "14990" uid: 51962a19-3458-46e0-a594-b7acf416cc16 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:25:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 5.193s (5.193s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-24T21:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:13Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d82ee470b6 namespace: kserve-ci-e2e-test resourceVersion: "15034" uid: 993b332b-4e47-4662-a02e-8eae6c672d48 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:25:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:13Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d8347c2d65 namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: dbb2dbe7-4099-479f-ac20-919d45d0272a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:25:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:13Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d836491c1c namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: 09aa1f39-7094-417a-b4c6-69df20a2ec55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-24T21:25:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:27:12Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:12Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556d8a210a0db namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 26259f61-9f7a-40b3-ac6b-dfcb6f9e0d65 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:27:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:11Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556f3a496ae31 namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: e21d2cc8-6050-4c3b-8c41-0a5b139cd6a6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:27:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:11Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556f3a49733cb namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: cbfe3743-606d-4bf9-8203-2c185a59a54e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-96447656d-6bksn namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: c2906e30-cc2e-4938-bf3d-476410e6a7e2 kind: Event lastTimestamp: "2026-07-24T21:27:17Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-24T21:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:17Z" name: isvc-sklearn-graph-2-predictor-96447656d-6bksn.18c556f50a083e7b namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 5f669c64-2c04-4966-8b97-d9f33903bf3f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-96447656d namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 40eb588c-2c48-4faf-9c9b-fd95f8bf7b44 kind: Event lastTimestamp: "2026-07-24T21:24:52Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-96447656d-6bksn' metadata: creationTimestamp: "2026-07-24T21:24:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:24:52Z" name: isvc-sklearn-graph-2-predictor-96447656d.18c556d3275a5958 namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 95a301ee-73ee-4394-a5ac-7e29ad0cb633 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: 25b58841-1071-4ab7-8204-0192df5336eb kind: Event lastTimestamp: "2026-07-24T21:24:52Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-96447656d from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:24:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:24:52Z" name: isvc-sklearn-graph-2-predictor.18c556d31f95cd48 namespace: kserve-ci-e2e-test resourceVersion: "14760" uid: 500c78f3-ba84-424c-89e9-94439d7c9741 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:25:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14756" uid: e84d9845-e083-4061-8f59-7f5797514213 kind: Event lastTimestamp: "2026-07-24T21:25:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:25:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:25:37Z" name: isvc-sklearn-graph-2-predictor.18c556d69e1a6765 namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: bcae6d98-702b-4947-9b7e-db92ae991593 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:25:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14756" uid: e84d9845-e083-4061-8f59-7f5797514213 kind: Event lastTimestamp: "2026-07-24T21:25:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:25:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:25:37Z" name: isvc-sklearn-graph-2-predictor.18c556d69e89ce4b namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 635d43c4-a11f-4339-934c-b4180bb45fce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:25:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: e84d9845-e083-4061-8f59-7f5797514213 kind: Event lastTimestamp: "2026-07-24T21:26:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:25:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:52Z" name: isvc-sklearn-graph-2-predictor.18c556e11a9b8ef3 namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: 4875287f-9531-49ac-ba55-f88a7f149940 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:25:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: e84d9845-e083-4061-8f59-7f5797514213 kind: Event lastTimestamp: "2026-07-24T21:26:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:25:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:52Z" name: isvc-sklearn-graph-2-predictor.18c556e11b059682 namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: f6460d55-5ee4-4f2c-b114-f96f3fdd83e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-24T21:24:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: 115b0846-cff4-4a95-ba10-7cac08f836c4 kind: Event lastTimestamp: "2026-07-24T21:26:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:24:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:26:33Z" name: isvc-sklearn-graph-2.18c556d31fc2e9a8 namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: 21ce82bd-d76e-4d3d-aecb-7b7746ce7ea7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: 115b0846-cff4-4a95-ba10-7cac08f836c4 kind: Event lastTimestamp: "2026-07-24T21:26:33Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-24T21:26:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:26:33Z" name: isvc-sklearn-graph-2.18c556eaa1076da0 namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: 9d119529-bfab-4514-8bef-1ef800d6dc47 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:24:52.035832Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-k52wj to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-07-24T21:24:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:24:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556d31c574269 namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 12fbfe17-8ae5-4301-a16f-5e59e0e5e5ff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:24:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:24:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556d32359ced4 namespace: kserve-ci-e2e-test resourceVersion: "14768" uid: d9625491-ca12-4f2b-87ed-fb93334ac63d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:53Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:24:53Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:24:53Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556d361b35f79 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 15f1f32d-701a-46f3-9ad0-14f6fb91ebf2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:24:53Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4bc99d27049df235a3e7474685f2095e74c6eed89dc3da2f17a1c3e85c3fa894" already present on machine metadata: creationTimestamp: "2026-07-24T21:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:53Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556d363a9a703 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 4d555163-49b4-4e53-bf60-f92964cbe669 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:24:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-24T21:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:53Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556d369dbde69 namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 04655d07-b112-43e7-afe2-500ffa43cb1a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:24:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-24T21:24:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:53Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556d36b0a2ed7 namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 32a8f1d7-db64-4a03-bf81-d6605298e5b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:24:58Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-24T21:24:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:58Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556d4a4dd8a1e namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: 128f580e-bc91-48bc-a86f-fd850061d7c4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:25:28Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 29.789s (29.789s including waiting). Image size: 1307355899 bytes.' metadata: creationTimestamp: "2026-07-24T21:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556db9474c810 namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: e8127ec8-244f-4c3e-bb25-a1012f904ab9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:25:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556db98151313 namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: 89934175-3bba-47f8-93b0-cde65dfcc57d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:25:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556db98a87c46 namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: 64f7c421-d3e5-42f4-8fb0-f233fc7734b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:25:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556db98b367c8 namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: 5f63eb07-5136-49f3-8541-090ad634b921 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:25:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556db9be9f2e4 namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: 879fb828-f5d3-4481-bc42-c23719623212 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:25:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556db9c68e555 namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: 77974b5d-38eb-42c7-83c9-9c7f3aa70113 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-24T21:25:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:26:25Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:25:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:25Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556dbe91d92ce namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: ae8abb68-344c-4024-bf6d-5d4e4d516c05 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:27:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556f3abf0eb71 namespace: kserve-ci-e2e-test resourceVersion: "16386" uid: 7ef3da16-8597-4ad2-9a71-9c037e2a804d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-k52wj namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 91c1b8bf-206a-4c81-8439-de3b355e3ec4 kind: Event lastTimestamp: "2026-07-24T21:27:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-k52wj.18c556f3abf166fb namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: 6585aa09-190c-464b-a7a9-206279b96323 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: 181f0cb3-4338-4596-8b2e-ea2b32c6c523 kind: Event lastTimestamp: "2026-07-24T21:24:52Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-k52wj' metadata: creationTimestamp: "2026-07-24T21:24:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:24:52Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c556d31bf2a2ed namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: 6b09fd56-5eba-4790-abb4-057f33d9bf45 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: 461a5e15-838f-4b7e-b8cf-a538719d4846 kind: Event lastTimestamp: "2026-07-24T21:24:51Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:24:51Z" name: isvc-xgboost-graph-predictor.18c556d318fd329e namespace: kserve-ci-e2e-test resourceVersion: "14733" uid: d1579e59-8cfa-49db-8d36-35eff71e1432 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-24T21:25:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: 264676ff-baba-4bfd-b0bf-12bc32fe201f kind: Event lastTimestamp: "2026-07-24T21:25:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:25:52Z" name: isvc-xgboost-graph-predictor.18c556d69763ff02 namespace: kserve-ci-e2e-test resourceVersion: "15375" uid: 54246fa3-b7d5-4e85-ac01-6b3035afc709 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-24T21:25:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: 264676ff-baba-4bfd-b0bf-12bc32fe201f kind: Event lastTimestamp: "2026-07-24T21:25:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:25:52Z" name: isvc-xgboost-graph-predictor.18c556d6979c9bb4 namespace: kserve-ci-e2e-test resourceVersion: "15376" uid: d3d31ccc-3b78-4d96-becf-a41671c4caf3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:26:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14964" uid: 264676ff-baba-4bfd-b0bf-12bc32fe201f kind: Event lastTimestamp: "2026-07-24T21:27:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:07Z" name: isvc-xgboost-graph-predictor.18c556e492736a01 namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 5bd67119-774f-4bac-8d6c-9ef61d7a666e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:26:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14964" uid: 264676ff-baba-4bfd-b0bf-12bc32fe201f kind: Event lastTimestamp: "2026-07-24T21:27:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:07Z" name: isvc-xgboost-graph-predictor.18c556e492f15ddf namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: ead0b1e0-eebd-4876-8458-6ae91d295530 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14714" uid: a73b7766-4b80-41b9-aa9a-4aeba16ed8bb kind: Event lastTimestamp: "2026-07-24T21:26:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:26:35Z" name: isvc-xgboost-graph.18c556d3190e54e7 namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: e3823feb-9ec4-48f1-85fd-93f5055709c3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: a73b7766-4b80-41b9-aa9a-4aeba16ed8bb kind: Event lastTimestamp: "2026-07-24T21:26:35Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-24T21:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:26:35Z" name: isvc-xgboost-graph.18c556eb4cca0b57 namespace: kserve-ci-e2e-test resourceVersion: "15941" uid: 6b4db91c-5e66-4866-97ed-0851165e035e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:27:01.531408Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-777d4d755d-5mb9s namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: c2376430-0dd9-49c4-9195-dd2871f56b3c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-777d4d755d-5mb9s to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:27:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:27:01Z" name: model-chainer-777d4d755d-5mb9s.18c556f142e0efc9 namespace: kserve-ci-e2e-test resourceVersion: "16173" uid: b08fae9a-b468-4731-8550-0915bfd3d48a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:01Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-777d4d755d-5mb9s namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: c2376430-0dd9-49c4-9195-dd2871f56b3c kind: Event lastTimestamp: "2026-07-24T21:27:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:01Z" name: model-chainer-777d4d755d-5mb9s.18c556f14d47ffb1 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: af28efb8-f8d5-4a56-b4ac-dbf8c000d58c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:02Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-777d4d755d-5mb9s namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: c2376430-0dd9-49c4-9195-dd2871f56b3c kind: Event lastTimestamp: "2026-07-24T21:27:02Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:27:02Z" name: model-chainer-777d4d755d-5mb9s.18c556f1808967fe namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: e1e48b6e-12e3-4470-92da-49dc2d5add60 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-777d4d755d-5mb9s namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: c2376430-0dd9-49c4-9195-dd2871f56b3c kind: Event lastTimestamp: "2026-07-24T21:27:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d388f8d2997cdb455a73f33beea2e1aa8fe959b1d5682a0690a87a2319ba2789" already present on machine metadata: creationTimestamp: "2026-07-24T21:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:02Z" name: model-chainer-777d4d755d-5mb9s.18c556f181c72b15 namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: ac800c30-6667-490d-b374-2a614fc4af77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-777d4d755d-5mb9s namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: c2376430-0dd9-49c4-9195-dd2871f56b3c kind: Event lastTimestamp: "2026-07-24T21:27:02Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-24T21:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:02Z" name: model-chainer-777d4d755d-5mb9s.18c556f184d912ed namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: a760df0f-dba2-4b90-ae2c-4f11921d0d12 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-777d4d755d-5mb9s namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: c2376430-0dd9-49c4-9195-dd2871f56b3c kind: Event lastTimestamp: "2026-07-24T21:27:02Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-24T21:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:02Z" name: model-chainer-777d4d755d-5mb9s.18c556f1854a672b namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 5e017f5c-dbfd-4edf-956c-d485538fa315 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-777d4d755d-5mb9s namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: c2376430-0dd9-49c4-9195-dd2871f56b3c kind: Event lastTimestamp: "2026-07-24T21:27:11Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:11Z" name: model-chainer-777d4d755d-5mb9s.18c556f39dd3935d namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: 739f826d-011e-412c-86de-e631bf2d4407 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-777d4d755d-5mb9s namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: c2376430-0dd9-49c4-9195-dd2871f56b3c kind: Event lastTimestamp: "2026-07-24T21:27:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-24T21:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:38Z" name: model-chainer-777d4d755d-5mb9s.18c556f42253cf95 namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: f662d901-c0a7-4f6e-a3e5-b07b0b047258 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-777d4d755d namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: a140ff53-3af9-4aa0-b161-9b812d3e1c54 kind: Event lastTimestamp: "2026-07-24T21:27:01Z" message: 'Created pod: model-chainer-777d4d755d-5mb9s' metadata: creationTimestamp: "2026-07-24T21:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:01Z" name: model-chainer-777d4d755d.18c556f1427501db namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 70ce24d5-6ca9-4f51-9f2c-bbf7166f4ea3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: e2bf1b46-41e1-4df3-804f-0a302a6d3f24 kind: Event lastTimestamp: "2026-07-24T21:27:01Z" message: Scaled up replica set model-chainer-777d4d755d from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:01Z" name: model-chainer.18c556f14189e272 namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: 26cf8276-ba05-46d8-8bbb-bae4083cdc52 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: e8ea3f72-c6c9-4a3c-9f33-3aff455db7cb kind: Event lastTimestamp: "2026-07-24T21:27:01Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-24T21:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:27:01Z" name: model-chainer.18c556f1467535f3 namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 8af7deba-b0a8-4c0f-bd2a-748e5116b6b2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:38:37.241041Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-59fd1-566c4755cc-q4lp4 namespace: kserve-ci-e2e-test resourceVersion: "22270" uid: ff8de588-b741-4b01-a4bd-bafee823b3fc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-59fd1-566c4755cc-q4lp4 to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:38:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:38:37Z" name: sequence-graph-59fd1-566c4755cc-q4lp4.18c557933e678c61 namespace: kserve-ci-e2e-test resourceVersion: "22275" uid: d5695cb9-08b8-4f67-a56a-e5ef862a381b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:38:37Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-59fd1-566c4755cc-q4lp4 namespace: kserve-ci-e2e-test resourceVersion: "22279" uid: ff8de588-b741-4b01-a4bd-bafee823b3fc kind: Event lastTimestamp: "2026-07-24T21:38:37Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:38:37Z" name: sequence-graph-59fd1-566c4755cc-q4lp4.18c55793590bbc53 namespace: kserve-ci-e2e-test resourceVersion: "22293" uid: 73138d87-c19e-49c9-841c-f304c2d9b5e3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:38:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-59fd1} kind: Pod name: sequence-graph-59fd1-566c4755cc-q4lp4 namespace: kserve-ci-e2e-test resourceVersion: "22274" uid: ff8de588-b741-4b01-a4bd-bafee823b3fc kind: Event lastTimestamp: "2026-07-24T21:38:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d388f8d2997cdb455a73f33beea2e1aa8fe959b1d5682a0690a87a2319ba2789" already present on machine metadata: creationTimestamp: "2026-07-24T21:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:38:37Z" name: sequence-graph-59fd1-566c4755cc-q4lp4.18c557935a7160ae namespace: kserve-ci-e2e-test resourceVersion: "22295" uid: 3606040e-0df2-47ed-82ca-dd1376619aec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:38:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-59fd1} kind: Pod name: sequence-graph-59fd1-566c4755cc-q4lp4 namespace: kserve-ci-e2e-test resourceVersion: "22274" uid: ff8de588-b741-4b01-a4bd-bafee823b3fc kind: Event lastTimestamp: "2026-07-24T21:38:37Z" message: 'Created container: sequence-graph-59fd1' metadata: creationTimestamp: "2026-07-24T21:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:38:37Z" name: sequence-graph-59fd1-566c4755cc-q4lp4.18c557935d7f6a24 namespace: kserve-ci-e2e-test resourceVersion: "22296" uid: abe34ba6-2121-4d7c-8d8e-ddc72718d6c8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:38:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-59fd1} kind: Pod name: sequence-graph-59fd1-566c4755cc-q4lp4 namespace: kserve-ci-e2e-test resourceVersion: "22274" uid: ff8de588-b741-4b01-a4bd-bafee823b3fc kind: Event lastTimestamp: "2026-07-24T21:38:37Z" message: Started container sequence-graph-59fd1 metadata: creationTimestamp: "2026-07-24T21:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:38:37Z" name: sequence-graph-59fd1-566c4755cc-q4lp4.18c557935deece22 namespace: kserve-ci-e2e-test resourceVersion: "22297" uid: c85cb2a2-cf6a-4517-9cf8-0b578c16385c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-59fd1} kind: Pod name: sequence-graph-59fd1-566c4755cc-q4lp4 namespace: kserve-ci-e2e-test resourceVersion: "22274" uid: ff8de588-b741-4b01-a4bd-bafee823b3fc kind: Event lastTimestamp: "2026-07-24T21:46:51Z" message: Stopping container sequence-graph-59fd1 metadata: creationTimestamp: "2026-07-24T21:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:51Z" name: sequence-graph-59fd1-566c4755cc-q4lp4.18c5580668d651c4 namespace: kserve-ci-e2e-test resourceVersion: "26202" uid: c2860412-d1b6-4e33-9f0b-424c9b3281c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-59fd1} kind: Pod name: sequence-graph-59fd1-566c4755cc-q4lp4 namespace: kserve-ci-e2e-test resourceVersion: "22274" uid: ff8de588-b741-4b01-a4bd-bafee823b3fc kind: Event lastTimestamp: "2026-07-24T21:47:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-24T21:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:19Z" name: sequence-graph-59fd1-566c4755cc-q4lp4.18c55806f198fae6 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: a7225fdc-d02b-44e4-a726-79b82c0aa59a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:38:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-59fd1-566c4755cc namespace: kserve-ci-e2e-test resourceVersion: "22265" uid: c0ee60de-bc6c-4ff1-8e70-09e566246ddf kind: Event lastTimestamp: "2026-07-24T21:38:37Z" message: 'Created pod: sequence-graph-59fd1-566c4755cc-q4lp4' metadata: creationTimestamp: "2026-07-24T21:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:37Z" name: sequence-graph-59fd1-566c4755cc.18c557933db10fdf namespace: kserve-ci-e2e-test resourceVersion: "22272" uid: a7614de8-1ccc-42f2-a73f-ee647b2c38ae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:38:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-59fd1 namespace: kserve-ci-e2e-test resourceVersion: "22262" uid: 94b4de64-44c4-4dbd-8775-16c951c15675 kind: Event lastTimestamp: "2026-07-24T21:38:37Z" message: Scaled up replica set sequence-graph-59fd1-566c4755cc from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:37Z" name: sequence-graph-59fd1.18c557933b9d1a84 namespace: kserve-ci-e2e-test resourceVersion: "22267" uid: b69346e3-7631-419c-af34-ec5fa2892fe9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:38:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-59fd1 namespace: kserve-ci-e2e-test resourceVersion: "22283" uid: 59d648c3-a3b9-4e4c-9c5b-98b8d9abf688 kind: Event lastTimestamp: "2026-07-24T21:38:37Z" message: InferenceGraph [sequence-graph-59fd1] is Ready metadata: creationTimestamp: "2026-07-24T21:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:38:37Z" name: sequence-graph-59fd1.18c5579342627216 namespace: kserve-ci-e2e-test resourceVersion: "22285" uid: 29ccd4dc-890e-45a8-b8bc-22375336ebb3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:38:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-59fd1 namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: c172270b-d2ff-424f-bc89-b0a2d193c763 kind: Event lastTimestamp: "2026-07-24T21:39:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:38:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:39:07Z" name: sequence-graph-59fd1.18c55796b990bd5d namespace: kserve-ci-e2e-test resourceVersion: "22513" uid: d0146de6-d752-46be-83d7-bea98d4b74f6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:38:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-59fd1 namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: c172270b-d2ff-424f-bc89-b0a2d193c763 kind: Event lastTimestamp: "2026-07-24T21:39:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:38:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:39:07Z" name: sequence-graph-59fd1.18c55796b9ffd00f namespace: kserve-ci-e2e-test resourceVersion: "22515" uid: 4f556379-f481-422d-9842-a46de3264518 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:28:11.856862Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c3da7-5dc8cb65f5-f9454 namespace: kserve-ci-e2e-test resourceVersion: "16998" uid: 9ecafcfe-aedb-48b1-be80-6eabd1329a4d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-c3da7-5dc8cb65f5-f9454 to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:28:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:28:11Z" name: sequence-graph-c3da7-5dc8cb65f5-f9454.18c55701a29a3fc1 namespace: kserve-ci-e2e-test resourceVersion: "17005" uid: 4a5dea49-15e9-444e-9877-390ac77b108d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:28:12Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c3da7-5dc8cb65f5-f9454 namespace: kserve-ci-e2e-test resourceVersion: "17001" uid: 9ecafcfe-aedb-48b1-be80-6eabd1329a4d kind: Event lastTimestamp: "2026-07-24T21:28:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-c3da7-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:28:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:28:12Z" name: sequence-graph-c3da7-5dc8cb65f5-f9454.18c55701ac479c1d namespace: kserve-ci-e2e-test resourceVersion: "17022" uid: 2625f65f-0a44-422a-82ec-fb326fcfdf13 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:28:13Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c3da7-5dc8cb65f5-f9454 namespace: kserve-ci-e2e-test resourceVersion: "17009" uid: 9ecafcfe-aedb-48b1-be80-6eabd1329a4d kind: Event lastTimestamp: "2026-07-24T21:28:13Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:28:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:28:13Z" name: sequence-graph-c3da7-5dc8cb65f5-f9454.18c5570215e8671f namespace: kserve-ci-e2e-test resourceVersion: "17035" uid: 1e2d19d3-512f-43b0-af32-0b7f04ba5c63 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:28:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3da7} kind: Pod name: sequence-graph-c3da7-5dc8cb65f5-f9454 namespace: kserve-ci-e2e-test resourceVersion: "17001" uid: 9ecafcfe-aedb-48b1-be80-6eabd1329a4d kind: Event lastTimestamp: "2026-07-24T21:28:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d388f8d2997cdb455a73f33beea2e1aa8fe959b1d5682a0690a87a2319ba2789" already present on machine metadata: creationTimestamp: "2026-07-24T21:28:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:28:13Z" name: sequence-graph-c3da7-5dc8cb65f5-f9454.18c55702173fc468 namespace: kserve-ci-e2e-test resourceVersion: "17037" uid: b33de758-7de5-4965-8663-1dd2fec355fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:28:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3da7} kind: Pod name: sequence-graph-c3da7-5dc8cb65f5-f9454 namespace: kserve-ci-e2e-test resourceVersion: "17001" uid: 9ecafcfe-aedb-48b1-be80-6eabd1329a4d kind: Event lastTimestamp: "2026-07-24T21:28:13Z" message: 'Created container: sequence-graph-c3da7' metadata: creationTimestamp: "2026-07-24T21:28:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:28:13Z" name: sequence-graph-c3da7-5dc8cb65f5-f9454.18c557021b34e7db namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 4c0d5526-c89e-4e8b-965f-3ba2f5db5b5c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:28:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3da7} kind: Pod name: sequence-graph-c3da7-5dc8cb65f5-f9454 namespace: kserve-ci-e2e-test resourceVersion: "17001" uid: 9ecafcfe-aedb-48b1-be80-6eabd1329a4d kind: Event lastTimestamp: "2026-07-24T21:28:13Z" message: Started container sequence-graph-c3da7 metadata: creationTimestamp: "2026-07-24T21:28:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:28:13Z" name: sequence-graph-c3da7-5dc8cb65f5-f9454.18c557021ba8f33a namespace: kserve-ci-e2e-test resourceVersion: "17039" uid: 8e9b77c7-dd61-4622-b032-2b0dcb2b8b02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3da7} kind: Pod name: sequence-graph-c3da7-5dc8cb65f5-f9454 namespace: kserve-ci-e2e-test resourceVersion: "17001" uid: 9ecafcfe-aedb-48b1-be80-6eabd1329a4d kind: Event lastTimestamp: "2026-07-24T21:36:26Z" message: Stopping container sequence-graph-c3da7 metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:26Z" name: sequence-graph-c3da7-5dc8cb65f5-f9454.18c55774cff46384 namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: aa0ed108-9244-4028-b62b-3ed246621658 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:36:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3da7} kind: Pod name: sequence-graph-c3da7-5dc8cb65f5-f9454 namespace: kserve-ci-e2e-test resourceVersion: "17001" uid: 9ecafcfe-aedb-48b1-be80-6eabd1329a4d kind: Event lastTimestamp: "2026-07-24T21:36:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-24T21:36:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:55Z" name: sequence-graph-c3da7-5dc8cb65f5-f9454.18c55775aa66a292 namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: 1b0074bc-2938-43d4-aca2-57657cdff83b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:28:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-c3da7-5dc8cb65f5 namespace: kserve-ci-e2e-test resourceVersion: "16995" uid: 280ce7af-0c35-4089-8be4-b82e849478c1 kind: Event lastTimestamp: "2026-07-24T21:28:11Z" message: 'Created pod: sequence-graph-c3da7-5dc8cb65f5-f9454' metadata: creationTimestamp: "2026-07-24T21:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:28:11Z" name: sequence-graph-c3da7-5dc8cb65f5.18c55701a2276a7e namespace: kserve-ci-e2e-test resourceVersion: "17000" uid: 7a089758-4624-4e0f-a488-1a2f2c1afc3f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:28:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-c3da7 namespace: kserve-ci-e2e-test resourceVersion: "16993" uid: 084f1599-454a-4f29-ba24-0571b2ae25a1 kind: Event lastTimestamp: "2026-07-24T21:28:11Z" message: Scaled up replica set sequence-graph-c3da7-5dc8cb65f5 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:28:11Z" name: sequence-graph-c3da7.18c55701a13a399b namespace: kserve-ci-e2e-test resourceVersion: "16996" uid: be41769c-546a-4d6b-a7c3-5e80e3d5a804 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:28:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c3da7 namespace: kserve-ci-e2e-test resourceVersion: "17011" uid: b1f975c7-6353-4e5e-8bb2-555c4fe932f4 kind: Event lastTimestamp: "2026-07-24T21:28:11Z" message: InferenceGraph [sequence-graph-c3da7] is Ready metadata: creationTimestamp: "2026-07-24T21:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:28:11Z" name: sequence-graph-c3da7.18c55701a56f3064 namespace: kserve-ci-e2e-test resourceVersion: "17013" uid: 524dd511-d106-42f9-aa42-7fdb89980760 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:28:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3da7 namespace: kserve-ci-e2e-test resourceVersion: "16994" uid: 2922f00e-5922-44bc-ada6-a42e2fd84d44 kind: Event lastTimestamp: "2026-07-24T21:28:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:28:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:28:26Z" name: sequence-graph-c3da7.18c557051fcc59df namespace: kserve-ci-e2e-test resourceVersion: "17133" uid: 46bdf9e5-b2ce-4999-a39a-4bb8a3b18ed2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:28:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3da7 namespace: kserve-ci-e2e-test resourceVersion: "16994" uid: 2922f00e-5922-44bc-ada6-a42e2fd84d44 kind: Event lastTimestamp: "2026-07-24T21:28:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:28:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:28:26Z" name: sequence-graph-c3da7.18c55705204e0d1d namespace: kserve-ci-e2e-test resourceVersion: "17135" uid: 1f5fc9ab-a35a-40b0-b739-3e6127973f92 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:28:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3da7 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: 2922f00e-5922-44bc-ada6-a42e2fd84d44 kind: Event lastTimestamp: "2026-07-24T21:28:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:28:41Z" name: sequence-graph-c3da7.18c557089efc4adb namespace: kserve-ci-e2e-test resourceVersion: "17243" uid: 475db4f3-c869-4b8d-9594-6d32c2952523 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:28:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3da7 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: 2922f00e-5922-44bc-ada6-a42e2fd84d44 kind: Event lastTimestamp: "2026-07-24T21:28:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:28:41Z" name: sequence-graph-c3da7.18c557089f6565ff namespace: kserve-ci-e2e-test resourceVersion: "17245" uid: 1aa1d335-bbbd-4c1a-b776-2b67be1a58cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:37:26.804904Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f761f-8d6759985-j4lvt namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: e149d103-691c-4e5c-ac97-f5051ec0dea5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f761f-8d6759985-j4lvt to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:37:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:37:26Z" name: sequence-graph-f761f-8d6759985-j4lvt.18c55782d8155513 namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: ec21385b-864a-4805-97f8-21bff30bcf8d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:27Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f761f-8d6759985-j4lvt namespace: kserve-ci-e2e-test resourceVersion: "21472" uid: e149d103-691c-4e5c-ac97-f5051ec0dea5 kind: Event lastTimestamp: "2026-07-24T21:37:27Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:37:27Z" name: sequence-graph-f761f-8d6759985-j4lvt.18c55782f21c3a80 namespace: kserve-ci-e2e-test resourceVersion: "21490" uid: 3fca0dcb-73e9-4cae-b3b3-4f2c44f8b822 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f761f} kind: Pod name: sequence-graph-f761f-8d6759985-j4lvt namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: e149d103-691c-4e5c-ac97-f5051ec0dea5 kind: Event lastTimestamp: "2026-07-24T21:37:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d388f8d2997cdb455a73f33beea2e1aa8fe959b1d5682a0690a87a2319ba2789" already present on machine metadata: creationTimestamp: "2026-07-24T21:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:27Z" name: sequence-graph-f761f-8d6759985-j4lvt.18c55782f40913e3 namespace: kserve-ci-e2e-test resourceVersion: "21492" uid: a40f68eb-c178-4774-b3ae-7f64c5689574 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f761f} kind: Pod name: sequence-graph-f761f-8d6759985-j4lvt namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: e149d103-691c-4e5c-ac97-f5051ec0dea5 kind: Event lastTimestamp: "2026-07-24T21:37:27Z" message: 'Created container: sequence-graph-f761f' metadata: creationTimestamp: "2026-07-24T21:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:27Z" name: sequence-graph-f761f-8d6759985-j4lvt.18c55782f705e584 namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: 6c84faea-356d-4d24-93fb-61af175c2ed0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f761f} kind: Pod name: sequence-graph-f761f-8d6759985-j4lvt namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: e149d103-691c-4e5c-ac97-f5051ec0dea5 kind: Event lastTimestamp: "2026-07-24T21:37:27Z" message: Started container sequence-graph-f761f metadata: creationTimestamp: "2026-07-24T21:37:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:27Z" name: sequence-graph-f761f-8d6759985-j4lvt.18c55782f78c578d namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: cbc6820c-4f17-4f01-bca8-f9e58dbae3bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f761f} kind: Pod name: sequence-graph-f761f-8d6759985-j4lvt namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: e149d103-691c-4e5c-ac97-f5051ec0dea5 kind: Event lastTimestamp: "2026-07-24T21:37:36Z" message: Stopping container sequence-graph-f761f metadata: creationTimestamp: "2026-07-24T21:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:36Z" name: sequence-graph-f761f-8d6759985-j4lvt.18c557853187cb5b namespace: kserve-ci-e2e-test resourceVersion: "21600" uid: 84ea90f5-82c9-42f9-b532-0efb2816e990 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:37:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f761f} kind: Pod name: sequence-graph-f761f-8d6759985-j4lvt namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: e149d103-691c-4e5c-ac97-f5051ec0dea5 kind: Event lastTimestamp: "2026-07-24T21:38:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-24T21:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:38:03Z" name: sequence-graph-f761f-8d6759985-j4lvt.18c55785a5080acf namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: a27a726f-a461-424e-b8e7-e3e86103fd25 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f761f-8d6759985 namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: f925b4d7-bc84-4143-b2bd-859f1cd87dcd kind: Event lastTimestamp: "2026-07-24T21:37:26Z" message: 'Created pod: sequence-graph-f761f-8d6759985-j4lvt' metadata: creationTimestamp: "2026-07-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:26Z" name: sequence-graph-f761f-8d6759985.18c55782d7a31ef7 namespace: kserve-ci-e2e-test resourceVersion: "21464" uid: 855b1cd9-e9f7-44f2-8677-7f6d769ff9a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f761f namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: 4489ae67-0eed-4b19-881c-4bb70f0e4a5c kind: Event lastTimestamp: "2026-07-24T21:37:26Z" message: Scaled up replica set sequence-graph-f761f-8d6759985 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:26Z" name: sequence-graph-f761f.18c55782d6cf7f08 namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 41db7db9-1679-4905-be45-e9d95f538e34 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f761f namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 01530024-0dc7-4b65-9ada-1922a110dbcf kind: Event lastTimestamp: "2026-07-24T21:37:26Z" message: InferenceGraph [sequence-graph-f761f] is Ready metadata: creationTimestamp: "2026-07-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:26Z" name: sequence-graph-f761f.18c55782dcd18c8f namespace: kserve-ci-e2e-test resourceVersion: "21476" uid: abb18a36-693c-47ed-984a-c36db1de4f65 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f761f namespace: kserve-ci-e2e-test resourceVersion: "21452" uid: 01530024-0dc7-4b65-9ada-1922a110dbcf kind: Event lastTimestamp: "2026-07-24T21:37:26Z" message: 'Failed to update status for InferenceGraph "sequence-graph-f761f": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-f761f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:26Z" name: sequence-graph-f761f.18c55782dfcb12b8 namespace: kserve-ci-e2e-test resourceVersion: "21480" uid: f1ca027f-d85b-4226-9218-57404464d272 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f761f namespace: kserve-ci-e2e-test resourceVersion: "21452" uid: 01530024-0dc7-4b65-9ada-1922a110dbcf kind: Event lastTimestamp: "2026-07-24T21:37:26Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-f761f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:26Z" name: sequence-graph-f761f.18c55782dfcb65fc namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: ff069d7c-9dcd-4254-a691-a18e07909215 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:47:02.189212Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9739a-647798446f-ffsxh namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 1897e613-6bd4-4107-87ca-669289cd15a9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-9739a-647798446f-ffsxh to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:47:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:47:02Z" name: splitter-graph-9739a-647798446f-ffsxh.18c55808cfa91dff namespace: kserve-ci-e2e-test resourceVersion: "26429" uid: 764f79bf-7d84-42a4-933c-1293418e5f1e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:47:02Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9739a-647798446f-ffsxh namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: 1897e613-6bd4-4107-87ca-669289cd15a9 kind: Event lastTimestamp: "2026-07-24T21:47:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-9739a-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:02Z" name: splitter-graph-9739a-647798446f-ffsxh.18c55808da29b615 namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: 6dab4619-5211-414c-bf04-fd2fef5b6fac reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:04Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9739a-647798446f-ffsxh namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: 1897e613-6bd4-4107-87ca-669289cd15a9 kind: Event lastTimestamp: "2026-07-24T21:47:04Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:47:04Z" name: splitter-graph-9739a-647798446f-ffsxh.18c558094308d6e9 namespace: kserve-ci-e2e-test resourceVersion: "26459" uid: 1dd7ca54-ae5c-4503-9605-1ecb6f106bd0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9739a} kind: Pod name: splitter-graph-9739a-647798446f-ffsxh namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: 1897e613-6bd4-4107-87ca-669289cd15a9 kind: Event lastTimestamp: "2026-07-24T21:47:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d388f8d2997cdb455a73f33beea2e1aa8fe959b1d5682a0690a87a2319ba2789" already present on machine metadata: creationTimestamp: "2026-07-24T21:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:04Z" name: splitter-graph-9739a-647798446f-ffsxh.18c55809444b550b namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 639642ed-e266-4bbf-9317-084c1e1682ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9739a} kind: Pod name: splitter-graph-9739a-647798446f-ffsxh namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: 1897e613-6bd4-4107-87ca-669289cd15a9 kind: Event lastTimestamp: "2026-07-24T21:47:04Z" message: 'Created container: splitter-graph-9739a' metadata: creationTimestamp: "2026-07-24T21:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:04Z" name: splitter-graph-9739a-647798446f-ffsxh.18c5580947ba5106 namespace: kserve-ci-e2e-test resourceVersion: "26464" uid: f155cc78-5d33-4bc7-84fc-81077869c288 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9739a} kind: Pod name: splitter-graph-9739a-647798446f-ffsxh namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: 1897e613-6bd4-4107-87ca-669289cd15a9 kind: Event lastTimestamp: "2026-07-24T21:47:04Z" message: Started container splitter-graph-9739a metadata: creationTimestamp: "2026-07-24T21:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:04Z" name: splitter-graph-9739a-647798446f-ffsxh.18c55809482eabd0 namespace: kserve-ci-e2e-test resourceVersion: "26465" uid: 93f2d333-1e78-4d04-9c46-a6199f708b44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9739a} kind: Pod name: splitter-graph-9739a-647798446f-ffsxh namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: 1897e613-6bd4-4107-87ca-669289cd15a9 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: Stopping container splitter-graph-9739a metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:12Z" name: splitter-graph-9739a-647798446f-ffsxh.18c5580b294b3df1 namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: d697d465-073e-4f3c-a365-bf8283a56036 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9739a} kind: Pod name: splitter-graph-9739a-647798446f-ffsxh namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: 1897e613-6bd4-4107-87ca-669289cd15a9 kind: Event lastTimestamp: "2026-07-24T21:47:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-24T21:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:41Z" name: splitter-graph-9739a-647798446f-ffsxh.18c5580c27e21c50 namespace: kserve-ci-e2e-test resourceVersion: "26943" uid: cb54bf1d-f891-4c57-a4a8-6d99a60dbfc1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-9739a-647798446f namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: d9f4492f-2d0e-4b88-822c-cbefd0f5d8dc kind: Event lastTimestamp: "2026-07-24T21:47:02Z" message: 'Created pod: splitter-graph-9739a-647798446f-ffsxh' metadata: creationTimestamp: "2026-07-24T21:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:02Z" name: splitter-graph-9739a-647798446f.18c55808cf42fddc namespace: kserve-ci-e2e-test resourceVersion: "26424" uid: 72cf61d8-4ccd-40d2-9608-83318e62cf68 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-9739a namespace: kserve-ci-e2e-test resourceVersion: "26418" uid: 0efbcc3c-7070-475a-afa1-5888aeb9e8d8 kind: Event lastTimestamp: "2026-07-24T21:47:02Z" message: Scaled up replica set splitter-graph-9739a-647798446f from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:02Z" name: splitter-graph-9739a.18c55808ce804946 namespace: kserve-ci-e2e-test resourceVersion: "26422" uid: 5d43b98f-0f35-4010-bf77-80fabc6b9071 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-9739a namespace: kserve-ci-e2e-test resourceVersion: "26436" uid: 71c05c2d-4c08-4115-8aac-514374c6ee7d kind: Event lastTimestamp: "2026-07-24T21:47:02Z" message: InferenceGraph [splitter-graph-9739a] is Ready metadata: creationTimestamp: "2026-07-24T21:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:47:02Z" name: splitter-graph-9739a.18c55808d3f29ac9 namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: af4c0957-e9aa-4ff9-8943-a85d1c936124 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:48:12.510628Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-bce0e-8f65fd6fd-zlngh namespace: kserve-ci-e2e-test resourceVersion: "27257" uid: 1c9a9c53-5c0f-42d3-a45b-071d67b40be8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-bce0e-8f65fd6fd-zlngh to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:48:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:48:12Z" name: splitter-graph-bce0e-8f65fd6fd-zlngh.18c558192f24ca9b namespace: kserve-ci-e2e-test resourceVersion: "27262" uid: de5a2c1a-8ebe-4d98-b8b5-13e8ba9d8796 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:48:12Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-bce0e-8f65fd6fd-zlngh namespace: kserve-ci-e2e-test resourceVersion: "27266" uid: 1c9a9c53-5c0f-42d3-a45b-071d67b40be8 kind: Event lastTimestamp: "2026-07-24T21:48:12Z" message: Add eth0 [10.132.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:48:12Z" name: splitter-graph-bce0e-8f65fd6fd-zlngh.18c5581948c9f4a5 namespace: kserve-ci-e2e-test resourceVersion: "27279" uid: 9e1ae443-97fa-4b61-a17e-4fe9384b13c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bce0e} kind: Pod name: splitter-graph-bce0e-8f65fd6fd-zlngh namespace: kserve-ci-e2e-test resourceVersion: "27258" uid: 1c9a9c53-5c0f-42d3-a45b-071d67b40be8 kind: Event lastTimestamp: "2026-07-24T21:48:12Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d388f8d2997cdb455a73f33beea2e1aa8fe959b1d5682a0690a87a2319ba2789" already present on machine metadata: creationTimestamp: "2026-07-24T21:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:48:12Z" name: splitter-graph-bce0e-8f65fd6fd-zlngh.18c558194a0e6903 namespace: kserve-ci-e2e-test resourceVersion: "27281" uid: 90a48342-a64f-4b8a-9c34-ec154722b081 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:48:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bce0e} kind: Pod name: splitter-graph-bce0e-8f65fd6fd-zlngh namespace: kserve-ci-e2e-test resourceVersion: "27258" uid: 1c9a9c53-5c0f-42d3-a45b-071d67b40be8 kind: Event lastTimestamp: "2026-07-24T21:48:13Z" message: 'Created container: splitter-graph-bce0e' metadata: creationTimestamp: "2026-07-24T21:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:48:13Z" name: splitter-graph-bce0e-8f65fd6fd-zlngh.18c558194cf7055c namespace: kserve-ci-e2e-test resourceVersion: "27282" uid: 7d4aff52-e412-4789-a062-04b38b4fd655 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:48:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bce0e} kind: Pod name: splitter-graph-bce0e-8f65fd6fd-zlngh namespace: kserve-ci-e2e-test resourceVersion: "27258" uid: 1c9a9c53-5c0f-42d3-a45b-071d67b40be8 kind: Event lastTimestamp: "2026-07-24T21:48:13Z" message: Started container splitter-graph-bce0e metadata: creationTimestamp: "2026-07-24T21:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:48:13Z" name: splitter-graph-bce0e-8f65fd6fd-zlngh.18c558194d6a12cc namespace: kserve-ci-e2e-test resourceVersion: "27283" uid: 07ad4746-54be-4bff-8931-603cf7b788a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:56:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bce0e} kind: Pod name: splitter-graph-bce0e-8f65fd6fd-zlngh namespace: kserve-ci-e2e-test resourceVersion: "27258" uid: 1c9a9c53-5c0f-42d3-a45b-071d67b40be8 kind: Event lastTimestamp: "2026-07-24T21:56:27Z" message: Stopping container splitter-graph-bce0e metadata: creationTimestamp: "2026-07-24T21:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:56:27Z" name: splitter-graph-bce0e-8f65fd6fd-zlngh.18c5588c5d5a88c1 namespace: kserve-ci-e2e-test resourceVersion: "30727" uid: ff5db01f-cbd6-4bd8-80ef-a10cc49ff41e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:56:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bce0e} kind: Pod name: splitter-graph-bce0e-8f65fd6fd-zlngh namespace: kserve-ci-e2e-test resourceVersion: "27258" uid: 1c9a9c53-5c0f-42d3-a45b-071d67b40be8 kind: Event lastTimestamp: "2026-07-24T21:56:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-24T21:56:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:56:54Z" name: splitter-graph-bce0e-8f65fd6fd-zlngh.18c5588cfd612a65 namespace: kserve-ci-e2e-test resourceVersion: "30967" uid: 98f637d1-e06f-42f1-a63c-bc006435bc91 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:48:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-bce0e-8f65fd6fd namespace: kserve-ci-e2e-test resourceVersion: "27254" uid: 96ebf713-c770-401a-81da-1cde701871db kind: Event lastTimestamp: "2026-07-24T21:48:12Z" message: 'Created pod: splitter-graph-bce0e-8f65fd6fd-zlngh' metadata: creationTimestamp: "2026-07-24T21:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:12Z" name: splitter-graph-bce0e-8f65fd6fd.18c558192ebf5a19 namespace: kserve-ci-e2e-test resourceVersion: "27259" uid: a542c915-35ac-4035-af0e-badb4ed4b7da reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:48:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-bce0e namespace: kserve-ci-e2e-test resourceVersion: "27252" uid: 7dc706ef-981f-483d-a837-c4c4ab19d0af kind: Event lastTimestamp: "2026-07-24T21:48:12Z" message: Scaled up replica set splitter-graph-bce0e-8f65fd6fd from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:12Z" name: splitter-graph-bce0e.18c558192e019072 namespace: kserve-ci-e2e-test resourceVersion: "27255" uid: 45816668-4f14-4841-a826-2808f243ba32 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:48:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-bce0e namespace: kserve-ci-e2e-test resourceVersion: "27268" uid: 83db48c1-4a24-4511-85f0-5d832f4e82ba kind: Event lastTimestamp: "2026-07-24T21:48:12Z" message: InferenceGraph [splitter-graph-bce0e] is Ready metadata: creationTimestamp: "2026-07-24T21:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:48:12Z" name: splitter-graph-bce0e.18c5581933226200 namespace: kserve-ci-e2e-test resourceVersion: "27270" uid: 35c77082-ebaf-4eae-87f6-2e570ac36fc8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:48:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-bce0e namespace: kserve-ci-e2e-test resourceVersion: "27253" uid: 6a53e335-907d-43f4-a6e4-857bda424107 kind: Event lastTimestamp: "2026-07-24T21:48:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:27Z" name: splitter-graph-bce0e.18c5581cac57b2e4 namespace: kserve-ci-e2e-test resourceVersion: "27398" uid: b1ef08ff-fda9-4244-8348-1ee78db880f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:48:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-bce0e namespace: kserve-ci-e2e-test resourceVersion: "27253" uid: 6a53e335-907d-43f4-a6e4-857bda424107 kind: Event lastTimestamp: "2026-07-24T21:48:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:27Z" name: splitter-graph-bce0e.18c5581cacbc1533 namespace: kserve-ci-e2e-test resourceVersion: "27400" uid: d6707151-b753-4009-b128-01276811ceba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:48:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-bce0e namespace: kserve-ci-e2e-test resourceVersion: "27399" uid: 6a53e335-907d-43f4-a6e4-857bda424107 kind: Event lastTimestamp: "2026-07-24T21:48:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:42Z" name: splitter-graph-bce0e.18c558202c2bc0d9 namespace: kserve-ci-e2e-test resourceVersion: "27514" uid: dd4858ba-eed6-4f23-b941-af1162f78734 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:48:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-bce0e namespace: kserve-ci-e2e-test resourceVersion: "27399" uid: 6a53e335-907d-43f4-a6e4-857bda424107 kind: Event lastTimestamp: "2026-07-24T21:48:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:42Z" name: splitter-graph-bce0e.18c558202d06dce0 namespace: kserve-ci-e2e-test resourceVersion: "27515" uid: c699a41b-69cd-4ff3-a3c4-3519a14b120a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:37:37.098918Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26 namespace: kserve-ci-e2e-test resourceVersion: "21649" uid: 2ec2fea9-5fb1-49cd-a93d-0c328df00f87 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-59fd1-predictor-54b67dc744-8wc26 to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26.18c557853da78c14 namespace: kserve-ci-e2e-test resourceVersion: "21659" uid: 47026803-91ee-47da-9968-3672af9b0b7c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26 namespace: kserve-ci-e2e-test resourceVersion: "21663" uid: 2ec2fea9-5fb1-49cd-a93d-0c328df00f87 kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26.18c55785578244a2 namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: cb936b1c-3253-4732-aa27-8260db6244a5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 2ec2fea9-5fb1-49cd-a93d-0c328df00f87 kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26.18c557855905a342 namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: 72ed07f1-b2bc-4cfd-8327-74ebe4fb6c6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 2ec2fea9-5fb1-49cd-a93d-0c328df00f87 kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26.18c557855c6de1ee namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: 2203f7d4-1c39-45e6-bef7-3d8bb32299c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 2ec2fea9-5fb1-49cd-a93d-0c328df00f87 kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26.18c557855d193592 namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 2689e560-9040-4ea9-b6fd-2c8717276e44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 2ec2fea9-5fb1-49cd-a93d-0c328df00f87 kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26.18c557855d236952 namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 115804d4-79c9-4447-b38e-e0c2cd50c349 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 2ec2fea9-5fb1-49cd-a93d-0c328df00f87 kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26.18c5578560b74033 namespace: kserve-ci-e2e-test resourceVersion: "21711" uid: ede90b4b-7084-40d8-ab3d-9dfedafcceae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 2ec2fea9-5fb1-49cd-a93d-0c328df00f87 kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26.18c55785612aa412 namespace: kserve-ci-e2e-test resourceVersion: "21712" uid: 50a5bbda-4bfa-4cc8-b4e9-fe8f0159b9cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-24T21:37:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 2ec2fea9-5fb1-49cd-a93d-0c328df00f87 kind: Event lastTimestamp: "2026-07-24T21:46:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:54Z" name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26.18c557857f132c2a namespace: kserve-ci-e2e-test resourceVersion: "26344" uid: ff7fb759-e886-43bf-9a5b-ae11658bee86 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 2ec2fea9-5fb1-49cd-a93d-0c328df00f87 kind: Event lastTimestamp: "2026-07-24T21:46:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:51Z" name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26.18c558066fa8f0f2 namespace: kserve-ci-e2e-test resourceVersion: "26229" uid: 1767d02d-03e6-418d-ac2c-fc1bbf1791b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 2ec2fea9-5fb1-49cd-a93d-0c328df00f87 kind: Event lastTimestamp: "2026-07-24T21:46:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:51Z" name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26.18c558066fa9faab namespace: kserve-ci-e2e-test resourceVersion: "26232" uid: 242fc3b6-4a84-4adf-b49d-ac802180daa7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 2ec2fea9-5fb1-49cd-a93d-0c328df00f87 kind: Event lastTimestamp: "2026-07-24T21:46:54Z" message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp 10.133.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:54Z" name: success-200-isvc-59fd1-predictor-54b67dc744-8wc26.18c55806f39807ca namespace: kserve-ci-e2e-test resourceVersion: "26343" uid: 817de803-2465-41ef-95de-0629b59a8b1b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-59fd1-predictor-54b67dc744 namespace: kserve-ci-e2e-test resourceVersion: "21635" uid: 3578b72b-0cd6-4860-a081-b770e305f744 kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: 'Created pod: success-200-isvc-59fd1-predictor-54b67dc744-8wc26' metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-59fd1-predictor-54b67dc744.18c557853d0c00a4 namespace: kserve-ci-e2e-test resourceVersion: "21651" uid: bd06938f-2bcf-435a-b023-b734936f5af1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-59fd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: 15846968-ebc9-4ac1-b0e0-4b2bb79c9671 kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Scaled up replica set success-200-isvc-59fd1-predictor-54b67dc744 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-59fd1-predictor.18c557853b27b87c namespace: kserve-ci-e2e-test resourceVersion: "21637" uid: 32030a92-23de-4f08-96ae-d1d937d778d8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:37:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-59fd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21634" uid: 9898c7f9-64a0-4f78-9ff0-e7531103c38f kind: Event lastTimestamp: "2026-07-24T21:38:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:07Z" name: success-200-isvc-59fd1-predictor.18c55788b9d81439 namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: a3d681c5-0d4b-4074-b568-831fb5d2aff8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:37:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-59fd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21634" uid: 9898c7f9-64a0-4f78-9ff0-e7531103c38f kind: Event lastTimestamp: "2026-07-24T21:38:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:37:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:07Z" name: success-200-isvc-59fd1-predictor.18c55788ba1d2e27 namespace: kserve-ci-e2e-test resourceVersion: "22017" uid: eccb4bdb-c7fb-4316-b146-8e8d8d1f41e8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:38:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-59fd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21891" uid: 9898c7f9-64a0-4f78-9ff0-e7531103c38f kind: Event lastTimestamp: "2026-07-24T21:38:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:52Z" name: success-200-isvc-59fd1-predictor.18c5578fb94989c4 namespace: kserve-ci-e2e-test resourceVersion: "22405" uid: 6a77a34f-e19b-4297-b932-61a8fde50c92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:38:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-59fd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21891" uid: 9898c7f9-64a0-4f78-9ff0-e7531103c38f kind: Event lastTimestamp: "2026-07-24T21:38:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:52Z" name: success-200-isvc-59fd1-predictor.18c5578fb9dbd9fd namespace: kserve-ci-e2e-test resourceVersion: "22406" uid: 88a49d15-c628-4846-84cd-6146d180a1fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-59fd1 namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: 431c617e-88ff-4bb7-bc93-1cbeef4997ce kind: Event lastTimestamp: "2026-07-24T21:38:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:38:24Z" name: success-200-isvc-59fd1.18c557853b70aa15 namespace: kserve-ci-e2e-test resourceVersion: "22157" uid: 0193eaf0-9d34-49ae-81d7-a3a7f9f2222e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-59fd1 namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: 431c617e-88ff-4bb7-bc93-1cbeef4997ce kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: 'Failed to update status for InferenceService "success-200-isvc-59fd1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-59fd1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-59fd1.18c557853be1d605 namespace: kserve-ci-e2e-test resourceVersion: "21645" uid: 3d4db6fe-e3a5-4c35-8c8e-b2e8f87a6280 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-59fd1 namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: 431c617e-88ff-4bb7-bc93-1cbeef4997ce kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-59fd1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-59fd1.18c557853be218e5 namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: c38eef95-8191-4801-9888-e7499bf21d50 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:38:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-59fd1 namespace: kserve-ci-e2e-test resourceVersion: "22153" uid: 431c617e-88ff-4bb7-bc93-1cbeef4997ce kind: Event lastTimestamp: "2026-07-24T21:38:24Z" message: InferenceService [success-200-isvc-59fd1] is Ready metadata: creationTimestamp: "2026-07-24T21:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:38:24Z" name: success-200-isvc-59fd1.18c557903e36d1b8 namespace: kserve-ci-e2e-test resourceVersion: "22155" uid: 490838e7-7b0a-4a3e-9de9-fe8bf1d699e1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:36:47.192995Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx namespace: kserve-ci-e2e-test resourceVersion: "21026" uid: cb2b02f6-5377-4a12-bab2-fb98f929fb23 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6de22-predictor-b49bb5776-zh2fx to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:36:47Z" name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx.18c557799f07974f namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: 83a61aac-e036-4312-9f9d-d22ec9a5bb4b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx namespace: kserve-ci-e2e-test resourceVersion: "21044" uid: cb2b02f6-5377-4a12-bab2-fb98f929fb23 kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:36:47Z" name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx.18c55779b8fc433a namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: bc516a39-7afd-4002-8928-825c92a768b9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: cb2b02f6-5377-4a12-bab2-fb98f929fb23 kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:47Z" name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx.18c55779ba7b8022 namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: 5b3f1644-5801-4de6-b197-df2ace625e9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: cb2b02f6-5377-4a12-bab2-fb98f929fb23 kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:47Z" name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx.18c55779bd6e1c1f namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: 48fdbb94-c09b-477d-a393-abb3ef6205b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: cb2b02f6-5377-4a12-bab2-fb98f929fb23 kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:47Z" name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx.18c55779bde57c65 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: b26a675d-0512-454c-891f-99b1db7b2e3a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: cb2b02f6-5377-4a12-bab2-fb98f929fb23 kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:47Z" name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx.18c55779bdef306f namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: 5ebcf925-3d00-4a63-a5c0-89491dc1fc54 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: cb2b02f6-5377-4a12-bab2-fb98f929fb23 kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:47Z" name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx.18c55779c0878537 namespace: kserve-ci-e2e-test resourceVersion: "21088" uid: 94f52c09-ee43-4ac2-bf5d-8413865a9bed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: cb2b02f6-5377-4a12-bab2-fb98f929fb23 kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:47Z" name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx.18c55779c0f901d6 namespace: kserve-ci-e2e-test resourceVersion: "21090" uid: 21910df5-d2f2-4fe9-9e69-66642cecf15f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: cb2b02f6-5377-4a12-bab2-fb98f929fb23 kind: Event lastTimestamp: "2026-07-24T21:46:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:03Z" name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx.18c55779ccd16ff6 namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: dc42797a-94b5-4692-afed-589bff779f63 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: cb2b02f6-5377-4a12-bab2-fb98f929fb23 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx.18c557fad3a94600 namespace: kserve-ci-e2e-test resourceVersion: "25426" uid: f3e5e9e7-1c59-4eec-97c0-5287be071614 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: cb2b02f6-5377-4a12-bab2-fb98f929fb23 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx.18c557fad3a9b939 namespace: kserve-ci-e2e-test resourceVersion: "25432" uid: a48432ad-cdb7-4ba9-a813-3add4e7526a6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: cb2b02f6-5377-4a12-bab2-fb98f929fb23 kind: Event lastTimestamp: "2026-07-24T21:46:03Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:03Z" name: success-200-isvc-6de22-predictor-b49bb5776-zh2fx.18c557fb41355dff namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 58728f90-ae61-4962-b3ab-d4e9b82e3027 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6de22-predictor-b49bb5776 namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 72ba86ba-4acb-47f2-bf7d-c366c27fcc24 kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: 'Created pod: success-200-isvc-6de22-predictor-b49bb5776-zh2fx' metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:47Z" name: success-200-isvc-6de22-predictor-b49bb5776.18c557799e67d5ad namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: cff9f20a-0022-4643-b701-280d50346940 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6de22-predictor namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: 4372ae99-6326-413f-bec3-6b8cabcc124c kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: Scaled up replica set success-200-isvc-6de22-predictor-b49bb5776 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:47Z" name: success-200-isvc-6de22-predictor.18c557799bf5585a namespace: kserve-ci-e2e-test resourceVersion: "21016" uid: 0c9143bf-2b0e-43f0-94f0-c4da8c6a3356 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:37:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6de22-predictor namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 88518279-920c-4d57-b184-01e97424d896 kind: Event lastTimestamp: "2026-07-24T21:37:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:32Z" name: success-200-isvc-6de22-predictor.18c5577d19f4d30d namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: 5f41a3b4-0550-4ea6-b02e-684b08b97bbd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:37:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6de22-predictor namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 88518279-920c-4d57-b184-01e97424d896 kind: Event lastTimestamp: "2026-07-24T21:37:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:32Z" name: success-200-isvc-6de22-predictor.18c5577d1a8dc8a5 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 838dbb2d-841e-4071-bbf0-5ed1b9eaa520 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:37:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6de22-predictor namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: 88518279-920c-4d57-b184-01e97424d896 kind: Event lastTimestamp: "2026-07-24T21:38:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:02Z" name: success-200-isvc-6de22-predictor.18c5578796f779f1 namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: 05511044-92bd-4d7d-9082-ad505e1a87db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:37:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6de22-predictor namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: 88518279-920c-4d57-b184-01e97424d896 kind: Event lastTimestamp: "2026-07-24T21:38:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:37:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:38:02Z" name: success-200-isvc-6de22-predictor.18c55787978b0c17 namespace: kserve-ci-e2e-test resourceVersion: "21973" uid: e88b66e6-2571-4e30-a5f4-7e41e7c6f724 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6de22 namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: 8dd9dbd7-0fbf-43e2-a878-5922c619c718 kind: Event lastTimestamp: "2026-07-24T21:37:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:34Z" name: success-200-isvc-6de22.18c557799ba5368b namespace: kserve-ci-e2e-test resourceVersion: "21556" uid: c4a941d1-dd5b-412b-b3cd-a7d2ab4ae1c3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6de22 namespace: kserve-ci-e2e-test resourceVersion: "21554" uid: 8dd9dbd7-0fbf-43e2-a878-5922c619c718 kind: Event lastTimestamp: "2026-07-24T21:37:34Z" message: InferenceService [success-200-isvc-6de22] is Ready metadata: creationTimestamp: "2026-07-24T21:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:34Z" name: success-200-isvc-6de22.18c5578487e1a0f6 namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 7b4c3605-260c-428c-a200-fd4abc046187 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:24:51.497898Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb namespace: kserve-ci-e2e-test resourceVersion: "14648" uid: bb1a3190-0dee-4710-880c-f3e577d60a83 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:24:51Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb.18c556d2fc470ac9 namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: 5dd5049d-f968-4cd1-ba3d-bc0e72a91182 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb namespace: kserve-ci-e2e-test resourceVersion: "14658" uid: bb1a3190-0dee-4710-880c-f3e577d60a83 kind: Event lastTimestamp: "2026-07-24T21:24:51Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:24:51Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb.18c556d316ada1bb namespace: kserve-ci-e2e-test resourceVersion: "14721" uid: 82d6a252-0b53-4c8d-838a-a5af2b463076 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: bb1a3190-0dee-4710-880c-f3e577d60a83 kind: Event lastTimestamp: "2026-07-24T21:24:51Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1716" metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:24:51Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb.18c556d3187db742 namespace: kserve-ci-e2e-test resourceVersion: "14731" uid: 76a28082-f465-4d9d-a157-24c032afe116 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: bb1a3190-0dee-4710-880c-f3e577d60a83 kind: Event lastTimestamp: "2026-07-24T21:25:08Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1716" in 16.524s (16.524s including waiting). Image size: 1334926948 bytes.' metadata: creationTimestamp: "2026-07-24T21:25:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:08Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb.18c556d6f1705afa namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 0f420dbc-fa46-4f3a-a9d9-e6e7d9dad545 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: bb1a3190-0dee-4710-880c-f3e577d60a83 kind: Event lastTimestamp: "2026-07-24T21:25:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:25:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:08Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb.18c556d6f54218ba namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: 6c8c62d9-69d5-4554-aebf-0c8afb3d8000 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: bb1a3190-0dee-4710-880c-f3e577d60a83 kind: Event lastTimestamp: "2026-07-24T21:25:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:25:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:08Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb.18c556d6f5d66052 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: dbf98a14-1aaa-45a5-91f8-7b2450e7152d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: bb1a3190-0dee-4710-880c-f3e577d60a83 kind: Event lastTimestamp: "2026-07-24T21:25:08Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-24T21:25:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:08Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb.18c556d6f6045138 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: d5ac027e-0566-4c69-983b-8c1d80f56541 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: bb1a3190-0dee-4710-880c-f3e577d60a83 kind: Event lastTimestamp: "2026-07-24T21:25:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 5.258s (5.258s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-24T21:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:13Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb.18c556d82f758907 namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: 408540e1-b0fd-4b89-8722-4f0ee042759e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: bb1a3190-0dee-4710-880c-f3e577d60a83 kind: Event lastTimestamp: "2026-07-24T21:25:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:13Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb.18c556d837a83282 namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: 2da2d807-851f-4eeb-8301-a466d7f34275 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:25:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: bb1a3190-0dee-4710-880c-f3e577d60a83 kind: Event lastTimestamp: "2026-07-24T21:25:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:25:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:13Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb.18c556d839473747 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 2db05528-9e30-431e-9662-c0a12063aba2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: bb1a3190-0dee-4710-880c-f3e577d60a83 kind: Event lastTimestamp: "2026-07-24T21:25:50Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:25:50Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb.18c556d865d2c865 namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: cba37e92-7371-48a6-9164-52bb2e9ab003 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: bb1a3190-0dee-4710-880c-f3e577d60a83 kind: Event lastTimestamp: "2026-07-24T21:26:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:21Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb.18c556e802a30fb3 namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: ef74f5d9-00d9-4b53-9c8b-574f2424a0ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: bb1a3190-0dee-4710-880c-f3e577d60a83 kind: Event lastTimestamp: "2026-07-24T21:26:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:21Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb.18c556e802a37e91 namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: 787d8e7f-9638-40aa-8c70-d780356ebda3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-77ecc-predictor-86d6bd8d6d namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: ca9d3cd6-524d-4c5e-b437-7fc7b3a31627 kind: Event lastTimestamp: "2026-07-24T21:24:51Z" message: 'Created pod: success-200-isvc-77ecc-predictor-86d6bd8d6d-25ldb' metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:24:51Z" name: success-200-isvc-77ecc-predictor-86d6bd8d6d.18c556d2fbddfde0 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: cdb5c24b-4219-42c7-bea7-d517714761c1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-77ecc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: c8119749-aa5f-4cf3-81e7-e6c60f916c7e kind: Event lastTimestamp: "2026-07-24T21:24:51Z" message: Scaled up replica set success-200-isvc-77ecc-predictor-86d6bd8d6d from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:24:51Z" name: success-200-isvc-77ecc-predictor.18c556d2f827743b namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: 6a4fd518-eb03-43fb-bdfa-a1d10575db04 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:25:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-77ecc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: bb47aae2-52ae-4153-958e-28ebb9eeeafe kind: Event lastTimestamp: "2026-07-24T21:25:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:25:36Z" name: success-200-isvc-77ecc-predictor.18c556d67d9787ea namespace: kserve-ci-e2e-test resourceVersion: "15242" uid: 03024c5a-5c99-4a87-be9a-61111b490b81 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:25:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-77ecc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: bb47aae2-52ae-4153-958e-28ebb9eeeafe kind: Event lastTimestamp: "2026-07-24T21:25:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:25:36Z" name: success-200-isvc-77ecc-predictor.18c556d67de769e3 namespace: kserve-ci-e2e-test resourceVersion: "15243" uid: a4132d44-3744-4411-a50a-a4fb63b1a44c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:25:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-77ecc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: bb47aae2-52ae-4153-958e-28ebb9eeeafe kind: Event lastTimestamp: "2026-07-24T21:26:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:25:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:06Z" name: success-200-isvc-77ecc-predictor.18c556e0fa93f3b2 namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: f24866eb-5643-4f98-86d7-0e16c9740c4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:25:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-77ecc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: bb47aae2-52ae-4153-958e-28ebb9eeeafe kind: Event lastTimestamp: "2026-07-24T21:26:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:25:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:06Z" name: success-200-isvc-77ecc-predictor.18c556e0faf81d12 namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: abbffa4a-d9c4-4153-b762-32d1a576e16e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:24:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-77ecc namespace: kserve-ci-e2e-test resourceVersion: "14632" uid: 79b05f9b-6e6e-47c0-a9f6-ca0b05d81a68 kind: Event lastTimestamp: "2026-07-24T21:26:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:26:00Z" name: success-200-isvc-77ecc.18c556d2fe495d9e namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: 2500f4fd-7f6d-467f-8f2b-c47ae879acd8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-77ecc namespace: kserve-ci-e2e-test resourceVersion: "15456" uid: 79b05f9b-6e6e-47c0-a9f6-ca0b05d81a68 kind: Event lastTimestamp: "2026-07-24T21:26:00Z" message: InferenceService [success-200-isvc-77ecc] is Ready metadata: creationTimestamp: "2026-07-24T21:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:26:00Z" name: success-200-isvc-77ecc.18c556e3212c7610 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 11074394-e14b-4abc-907c-26c3c2639171 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:46:02.204294Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8 namespace: kserve-ci-e2e-test resourceVersion: "25452" uid: 7b389104-8e8a-4de4-960b-10bf8781d481 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8 to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8.18c557fad847e0ba namespace: kserve-ci-e2e-test resourceVersion: "25457" uid: 5320e851-1b15-4897-98d5-e50b2d118c23 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8 namespace: kserve-ci-e2e-test resourceVersion: "25468" uid: 7b389104-8e8a-4de4-960b-10bf8781d481 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8.18c557faf200334c namespace: kserve-ci-e2e-test resourceVersion: "25512" uid: cecdf55d-abae-48f5-9ee4-209bbe64d26d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 7b389104-8e8a-4de4-960b-10bf8781d481 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8.18c557faf3409c6b namespace: kserve-ci-e2e-test resourceVersion: "25514" uid: 887a2a4f-b34e-4e37-8219-ebb11776ba88 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 7b389104-8e8a-4de4-960b-10bf8781d481 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8.18c557faf689d7c6 namespace: kserve-ci-e2e-test resourceVersion: "25516" uid: 0fa987f4-b4e7-46af-9b27-2268fe2247b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 7b389104-8e8a-4de4-960b-10bf8781d481 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8.18c557faf6fcce92 namespace: kserve-ci-e2e-test resourceVersion: "25518" uid: d47d3595-42d3-4976-a02e-b334f711c9aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 7b389104-8e8a-4de4-960b-10bf8781d481 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8.18c557faf70605ee namespace: kserve-ci-e2e-test resourceVersion: "25519" uid: 6a91c559-15a2-4de1-b2f6-8d8280fa17be reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 7b389104-8e8a-4de4-960b-10bf8781d481 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8.18c557fafab3974a namespace: kserve-ci-e2e-test resourceVersion: "25523" uid: 8b3e5b5f-eb60-4033-bddc-42b0230f9c10 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 7b389104-8e8a-4de4-960b-10bf8781d481 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8.18c557fafb273720 namespace: kserve-ci-e2e-test resourceVersion: "25524" uid: b13b894b-7613-4915-a3dc-3f3970b6c223 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 7b389104-8e8a-4de4-960b-10bf8781d481 kind: Event lastTimestamp: "2026-07-24T21:46:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:39Z" name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8.18c557fb247c62a6 namespace: kserve-ci-e2e-test resourceVersion: "26077" uid: 969f100a-f7ed-4f23-87e0-bc2186044656 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 7b389104-8e8a-4de4-960b-10bf8781d481 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:12Z" name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8.18c5580b30c3e48d namespace: kserve-ci-e2e-test resourceVersion: "26601" uid: fe4c03f7-c4bc-42cf-b370-1f00bfd78d5a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 7b389104-8e8a-4de4-960b-10bf8781d481 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:12Z" name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8.18c5580b30c4338b namespace: kserve-ci-e2e-test resourceVersion: "26604" uid: 8ccb3b3d-f7c3-4a0a-b7eb-cfb295c8f213 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 7b389104-8e8a-4de4-960b-10bf8781d481 kind: Event lastTimestamp: "2026-07-24T21:47:14Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:47:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:14Z" name: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8.18c5580bacb19a9d namespace: kserve-ci-e2e-test resourceVersion: "26697" uid: 584a44c8-cf0f-4d12-9d20-ce6615a85ee8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9739a-predictor-9fcd7d8d4 namespace: kserve-ci-e2e-test resourceVersion: "25422" uid: c33d9fdf-decd-428a-ad49-11a833fc6914 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: 'Created pod: success-200-isvc-9739a-predictor-9fcd7d8d4-bfbq8' metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-9739a-predictor-9fcd7d8d4.18c557fad7d3d58b namespace: kserve-ci-e2e-test resourceVersion: "25453" uid: ab1bb85d-513a-4fdf-be1b-fd35514a3e18 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9739a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25418" uid: 4dcb5dcc-554b-48eb-b0a6-d0b795dec905 kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: Scaled up replica set success-200-isvc-9739a-predictor-9fcd7d8d4 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-9739a-predictor.18c557fad3b07312 namespace: kserve-ci-e2e-test resourceVersion: "25427" uid: 351a066b-6c06-4a0a-977f-93934df3a409 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9739a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25421" uid: 8adb75f9-283d-43f3-96ba-51c786e55981 kind: Event lastTimestamp: "2026-07-24T21:46:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:46:17Z" name: success-200-isvc-9739a-predictor.18c557fe52410a1f namespace: kserve-ci-e2e-test resourceVersion: "25896" uid: 277e16db-3248-44e5-8861-c3d39420d45a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9739a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25421" uid: 8adb75f9-283d-43f3-96ba-51c786e55981 kind: Event lastTimestamp: "2026-07-24T21:46:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:46:17Z" name: success-200-isvc-9739a-predictor.18c557fe52af4f62 namespace: kserve-ci-e2e-test resourceVersion: "25897" uid: ab883501-e8c1-4856-bce5-db58fec7434a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:46:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9739a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25895" uid: 8adb75f9-283d-43f3-96ba-51c786e55981 kind: Event lastTimestamp: "2026-07-24T21:47:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:02Z" name: success-200-isvc-9739a-predictor.18c55801d15b5303 namespace: kserve-ci-e2e-test resourceVersion: "26428" uid: 5af3d014-bf16-447d-aac7-f566b1a3667a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:46:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9739a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25895" uid: 8adb75f9-283d-43f3-96ba-51c786e55981 kind: Event lastTimestamp: "2026-07-24T21:47:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:02Z" name: success-200-isvc-9739a-predictor.18c55801d1c4a9d3 namespace: kserve-ci-e2e-test resourceVersion: "26430" uid: 0eb56753-c771-4379-914b-f7968480e561 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9739a namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: a1b69896-92d6-4988-b8fe-ece155dd3d5e kind: Event lastTimestamp: "2026-07-24T21:46:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:46:49Z" name: success-200-isvc-9739a.18c557fad3e2068c namespace: kserve-ci-e2e-test resourceVersion: "26161" uid: 73341f08-b997-4f77-830f-ddf7e41faff5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9739a namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: a1b69896-92d6-4988-b8fe-ece155dd3d5e kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: 'Failed to update status for InferenceService "success-200-isvc-9739a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9739a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-9739a.18c557fad4ae1f78 namespace: kserve-ci-e2e-test resourceVersion: "25433" uid: 5adf46b6-15a1-470e-8e88-06a857b05528 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9739a namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: a1b69896-92d6-4988-b8fe-ece155dd3d5e kind: Event lastTimestamp: "2026-07-24T21:46:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9739a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:46:02Z" name: success-200-isvc-9739a.18c557fad4ae63f2 namespace: kserve-ci-e2e-test resourceVersion: "25437" uid: b5da6ce2-b293-4324-9f8a-0e666f178dcd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9739a namespace: kserve-ci-e2e-test resourceVersion: "26159" uid: a1b69896-92d6-4988-b8fe-ece155dd3d5e kind: Event lastTimestamp: "2026-07-24T21:46:49Z" message: InferenceService [success-200-isvc-9739a] is Ready metadata: creationTimestamp: "2026-07-24T21:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:46:49Z" name: success-200-isvc-9739a.18c55805e0760741 namespace: kserve-ci-e2e-test resourceVersion: "26160" uid: 4561d17f-23b3-47f3-80dd-f1e6623d6e90 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:46:52.035961Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: 751d85fc-12f4-44b5-8106-42e38a8743de kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a916f-predictor-698dc55bcb-b57tt to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:46:52Z" name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt.18c55806727ad1c2 namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 792868ed-bcec-4013-81a2-93033279c78b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 751d85fc-12f4-44b5-8106-42e38a8743de kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a916f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:52Z" name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt.18c558067f62c6c7 namespace: kserve-ci-e2e-test resourceVersion: "26297" uid: e5c24daa-982c-4694-a5de-358446171211 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:53Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt namespace: kserve-ci-e2e-test resourceVersion: "26251" uid: 751d85fc-12f4-44b5-8106-42e38a8743de kind: Event lastTimestamp: "2026-07-24T21:46:53Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:46:53Z" name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt.18c55806afe7905f namespace: kserve-ci-e2e-test resourceVersion: "26316" uid: 85539729-1a8c-4657-bb1a-72240f670f72 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 751d85fc-12f4-44b5-8106-42e38a8743de kind: Event lastTimestamp: "2026-07-24T21:46:53Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:53Z" name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt.18c55806b142ac88 namespace: kserve-ci-e2e-test resourceVersion: "26318" uid: 5c9fc064-cfd2-4990-92ed-ffea46aa51a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 751d85fc-12f4-44b5-8106-42e38a8743de kind: Event lastTimestamp: "2026-07-24T21:46:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:53Z" name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt.18c55806b457424d namespace: kserve-ci-e2e-test resourceVersion: "26321" uid: 54febd63-ef5b-458d-a370-4f074fa6a4c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 751d85fc-12f4-44b5-8106-42e38a8743de kind: Event lastTimestamp: "2026-07-24T21:46:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:53Z" name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt.18c55806b4ce127c namespace: kserve-ci-e2e-test resourceVersion: "26322" uid: 3618ca23-c4c9-48d5-aa82-d85b31aa5d9d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 751d85fc-12f4-44b5-8106-42e38a8743de kind: Event lastTimestamp: "2026-07-24T21:46:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:53Z" name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt.18c55806b4d708db namespace: kserve-ci-e2e-test resourceVersion: "26323" uid: f7cb49e7-c537-47e8-9380-456cd1b5af5c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 751d85fc-12f4-44b5-8106-42e38a8743de kind: Event lastTimestamp: "2026-07-24T21:46:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:53Z" name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt.18c55806b7dbf65d namespace: kserve-ci-e2e-test resourceVersion: "26325" uid: 47e5bd26-052b-4d85-b6d4-ef8371733af0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 751d85fc-12f4-44b5-8106-42e38a8743de kind: Event lastTimestamp: "2026-07-24T21:46:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:46:53Z" name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt.18c55806b84fcc6a namespace: kserve-ci-e2e-test resourceVersion: "26326" uid: 8458dc17-3ab6-49c2-8afd-439b5dbd8ca8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 751d85fc-12f4-44b5-8106-42e38a8743de kind: Event lastTimestamp: "2026-07-24T21:47:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:30Z" name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt.18c5580712b7968a namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 78aa0266-6e7b-4b0b-b22b-4daed5e59400 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T22:04:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 751d85fc-12f4-44b5-8106-42e38a8743de kind: Event lastTimestamp: "2026-07-24T22:04:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T22:04:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T22:04:11Z" name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt.18c558f87874c121 namespace: kserve-ci-e2e-test resourceVersion: "33959" uid: f058c9cc-3974-4457-a8e0-8ac8fcbad758 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T22:04:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 751d85fc-12f4-44b5-8106-42e38a8743de kind: Event lastTimestamp: "2026-07-24T22:04:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T22:04:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T22:04:11Z" name: success-200-isvc-a916f-predictor-698dc55bcb-b57tt.18c558f8787548a0 namespace: kserve-ci-e2e-test resourceVersion: "33960" uid: b3a6c2cc-11a3-48e4-8412-b34dcb83828a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a916f-predictor-698dc55bcb namespace: kserve-ci-e2e-test resourceVersion: "26231" uid: 7752df8a-04f9-4410-b6ab-8be98980b925 kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: 'Created pod: success-200-isvc-a916f-predictor-698dc55bcb-b57tt' metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:46:52Z" name: success-200-isvc-a916f-predictor-698dc55bcb.18c5580671fd1c7d namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 6726d779-fdca-47b8-ae92-0150bda6fba0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a916f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26227" uid: a8a187ec-bf71-447e-bf8e-ad449bad8302 kind: Event lastTimestamp: "2026-07-24T21:46:51Z" message: Scaled up replica set success-200-isvc-a916f-predictor-698dc55bcb from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:46:51Z" name: success-200-isvc-a916f-predictor.18c55806703f9d34 namespace: kserve-ci-e2e-test resourceVersion: "26233" uid: c07c55e0-7620-4c9f-a8ba-aee5ffd0894e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:47:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a916f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26230" uid: c17a2a4b-6b18-4444-b743-70c8ab9bd05d kind: Event lastTimestamp: "2026-07-24T21:47:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:47:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:22Z" name: success-200-isvc-a916f-predictor.18c55809eed38d8d namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: a5ef09c6-276a-4f5c-a583-1cb57c46c90b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:47:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a916f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26230" uid: c17a2a4b-6b18-4444-b743-70c8ab9bd05d kind: Event lastTimestamp: "2026-07-24T21:47:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:47:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:22Z" name: success-200-isvc-a916f-predictor.18c55809ef428c3f namespace: kserve-ci-e2e-test resourceVersion: "26771" uid: a4510e69-2173-4c7f-9f24-38203b2eb605 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:47:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a916f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26486" uid: c17a2a4b-6b18-4444-b743-70c8ab9bd05d kind: Event lastTimestamp: "2026-07-24T21:48:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:07Z" name: success-200-isvc-a916f-predictor.18c55810ed9ab11e namespace: kserve-ci-e2e-test resourceVersion: "27202" uid: 09a8de10-c713-4070-a39e-31a20a47deac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:47:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a916f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26486" uid: c17a2a4b-6b18-4444-b743-70c8ab9bd05d kind: Event lastTimestamp: "2026-07-24T21:48:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:07Z" name: success-200-isvc-a916f-predictor.18c55810edfb9971 namespace: kserve-ci-e2e-test resourceVersion: "27203" uid: 48bdbbc1-42c5-4d28-9370-d47301c94060 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:46:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a916f namespace: kserve-ci-e2e-test resourceVersion: "26207" uid: da7998fe-b94d-499a-8ef9-b1891fa21952 kind: Event lastTimestamp: "2026-07-24T21:47:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:47:40Z" name: success-200-isvc-a916f.18c558067052fa55 namespace: kserve-ci-e2e-test resourceVersion: "26939" uid: a360b7e7-749d-416e-8897-c06d83128411 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a916f namespace: kserve-ci-e2e-test resourceVersion: "26207" uid: da7998fe-b94d-499a-8ef9-b1891fa21952 kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: 'Failed to update status for InferenceService "success-200-isvc-a916f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a916f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:46:52Z" name: success-200-isvc-a916f.18c5580670e554c9 namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: 8d5354e1-dcde-4067-9cb2-f628ff54c157 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:46:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a916f namespace: kserve-ci-e2e-test resourceVersion: "26207" uid: da7998fe-b94d-499a-8ef9-b1891fa21952 kind: Event lastTimestamp: "2026-07-24T21:46:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a916f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:46:52Z" name: success-200-isvc-a916f.18c5580670e5adbd namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: 8941f801-5342-4bfd-9a77-f214e0aa4d8d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a916f namespace: kserve-ci-e2e-test resourceVersion: "26937" uid: da7998fe-b94d-499a-8ef9-b1891fa21952 kind: Event lastTimestamp: "2026-07-24T21:47:40Z" message: InferenceService [success-200-isvc-a916f] is Ready metadata: creationTimestamp: "2026-07-24T21:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:47:40Z" name: success-200-isvc-a916f.18c55811cdba9ea5 namespace: kserve-ci-e2e-test resourceVersion: "26938" uid: 77e8ffcd-1e83-40d1-8d25-b67404dd4911 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:35:36.822074Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7 namespace: kserve-ci-e2e-test resourceVersion: "20125" uid: a414b0d0-2b72-43b5-bcb2-723d0d0775ae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7 to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:35:36Z" name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7.18c557693c98885c namespace: kserve-ci-e2e-test resourceVersion: "20138" uid: d091be2c-8e8c-4aea-87b7-8b3368234d2d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7 namespace: kserve-ci-e2e-test resourceVersion: "20151" uid: a414b0d0-2b72-43b5-bcb2-723d0d0775ae kind: Event lastTimestamp: "2026-07-24T21:35:37Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:35:37Z" name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7.18c55769565f288b namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 95eb6e5c-7cbc-4a29-9299-946fbf3e0938 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7 namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: a414b0d0-2b72-43b5-bcb2-723d0d0775ae kind: Event lastTimestamp: "2026-07-24T21:35:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:37Z" name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7.18c5576957d5bd5b namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 65bbb9f5-79b3-495f-9183-0fcb12ce3f1d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7 namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: a414b0d0-2b72-43b5-bcb2-723d0d0775ae kind: Event lastTimestamp: "2026-07-24T21:35:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:37Z" name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7.18c557695a801424 namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: ed779688-294b-4b12-8033-8511bf91355b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7 namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: a414b0d0-2b72-43b5-bcb2-723d0d0775ae kind: Event lastTimestamp: "2026-07-24T21:35:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:37Z" name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7.18c557695af49f2e namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: ed74e6d4-3cb9-41e3-82f3-e08fd2d66efc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7 namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: a414b0d0-2b72-43b5-bcb2-723d0d0775ae kind: Event lastTimestamp: "2026-07-24T21:35:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:37Z" name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7.18c557695afda734 namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 21c6138a-0826-4b85-b94e-7a0d5c3461b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7 namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: a414b0d0-2b72-43b5-bcb2-723d0d0775ae kind: Event lastTimestamp: "2026-07-24T21:35:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:37Z" name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7.18c557695dcc2ff3 namespace: kserve-ci-e2e-test resourceVersion: "20195" uid: 0c77718b-492c-46c2-a818-0046c8f73499 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7 namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: a414b0d0-2b72-43b5-bcb2-723d0d0775ae kind: Event lastTimestamp: "2026-07-24T21:35:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:35:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:37Z" name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7.18c557695e40a6b9 namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: 431d0550-c28c-464b-90b6-fea9b900dc37 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:35:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7 namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: a414b0d0-2b72-43b5-bcb2-723d0d0775ae kind: Event lastTimestamp: "2026-07-24T21:36:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:14Z" name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7.18c55769a81b3568 namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: 99150d61-5cf6-4dff-bbd2-31511ee31278 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7 namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: a414b0d0-2b72-43b5-bcb2-723d0d0775ae kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:47Z" name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7.18c557799a8f7b80 namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: ab02d58a-0c55-4f38-b5e5-7d2dd757e10b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7 namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: a414b0d0-2b72-43b5-bcb2-723d0d0775ae kind: Event lastTimestamp: "2026-07-24T21:36:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:36:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:47Z" name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7.18c557799a90238b namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: 9b00089a-2c0e-467a-b74b-067ca655d9bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7 namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: a414b0d0-2b72-43b5-bcb2-723d0d0775ae kind: Event lastTimestamp: "2026-07-24T21:36:49Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:36:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:49Z" name: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7.18c5577a30371fe3 namespace: kserve-ci-e2e-test resourceVersion: "21123" uid: cc1bc19e-db99-4d3a-a06c-5b86f36657e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b6aba-predictor-75f57dcdb6 namespace: kserve-ci-e2e-test resourceVersion: "20111" uid: 414aaff8-cf9e-4f64-b990-804ad861660b kind: Event lastTimestamp: "2026-07-24T21:35:36Z" message: 'Created pod: success-200-isvc-b6aba-predictor-75f57dcdb6-tpnh7' metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:35:36Z" name: success-200-isvc-b6aba-predictor-75f57dcdb6.18c557693c14ee4d namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: 73cc4126-ff61-4cbf-88ca-b8fb316befee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b6aba-predictor namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 7bd79e71-a246-4215-82d0-997358c02627 kind: Event lastTimestamp: "2026-07-24T21:35:36Z" message: Scaled up replica set success-200-isvc-b6aba-predictor-75f57dcdb6 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:35:36Z" name: success-200-isvc-b6aba-predictor.18c5576938bc1120 namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: f9318ada-4a70-4572-9292-47154a86d5bb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:35:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b6aba-predictor namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 98f5e0ec-b8f0-4f8d-ba4e-ff94a90ebe35 kind: Event lastTimestamp: "2026-07-24T21:36:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:06Z" name: success-200-isvc-b6aba-predictor.18c5576cb7e98528 namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 88f3fb63-dcfe-47f2-af49-34e2096133f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:35:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b6aba-predictor namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 98f5e0ec-b8f0-4f8d-ba4e-ff94a90ebe35 kind: Event lastTimestamp: "2026-07-24T21:36:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:06Z" name: success-200-isvc-b6aba-predictor.18c5576cb870e43c namespace: kserve-ci-e2e-test resourceVersion: "20449" uid: b9cea430-5dc8-45d4-8c94-c33f8f6a433c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:36:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b6aba-predictor namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 98f5e0ec-b8f0-4f8d-ba4e-ff94a90ebe35 kind: Event lastTimestamp: "2026-07-24T21:36:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:36:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:36Z" name: success-200-isvc-b6aba-predictor.18c55773b671219c namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 6040af16-cb4d-453e-8adf-a34c6223f98a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:36:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b6aba-predictor namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 98f5e0ec-b8f0-4f8d-ba4e-ff94a90ebe35 kind: Event lastTimestamp: "2026-07-24T21:36:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:36:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:36Z" name: success-200-isvc-b6aba-predictor.18c55773b6c12081 namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: 3b2c086f-daa4-46b9-890f-261347345697 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b6aba namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: b546dcec-7816-4029-999c-e04b0dfe0141 kind: Event lastTimestamp: "2026-07-24T21:36:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:36:24Z" name: success-200-isvc-b6aba.18c5576938ca4aec namespace: kserve-ci-e2e-test resourceVersion: "20593" uid: 89862f85-8090-4275-b3cc-2ce55ef8153d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b6aba namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: b546dcec-7816-4029-999c-e04b0dfe0141 kind: Event lastTimestamp: "2026-07-24T21:35:36Z" message: 'Failed to update status for InferenceService "success-200-isvc-b6aba": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b6aba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:35:36Z" name: success-200-isvc-b6aba.18c55769396d1b9a namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 02c211ec-31ed-4b42-91bc-6e94a58e0b1d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b6aba namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: b546dcec-7816-4029-999c-e04b0dfe0141 kind: Event lastTimestamp: "2026-07-24T21:35:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b6aba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:35:36Z" name: success-200-isvc-b6aba.18c55769396d6011 namespace: kserve-ci-e2e-test resourceVersion: "20121" uid: d3a560ba-f422-46fa-a0cd-6a8905b227cb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b6aba namespace: kserve-ci-e2e-test resourceVersion: "20589" uid: b546dcec-7816-4029-999c-e04b0dfe0141 kind: Event lastTimestamp: "2026-07-24T21:36:24Z" message: InferenceService [success-200-isvc-b6aba] is Ready metadata: creationTimestamp: "2026-07-24T21:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:36:24Z" name: success-200-isvc-b6aba.18c557746662cb5c namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: c52e5d6d-2969-411f-b854-ce4d88f74b4c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:47:12.450979Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: 1596af97-aee1-48e6-bd5d-683b67ad2864 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bce0e-predictor-9867f5768-ttq7j to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:47:12Z" name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j.18c5580b334f48db namespace: kserve-ci-e2e-test resourceVersion: "26617" uid: 9471d16f-6c5a-4dbf-ae21-62ea46be6cda reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j namespace: kserve-ci-e2e-test resourceVersion: "26621" uid: 1596af97-aee1-48e6-bd5d-683b67ad2864 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:47:12Z" name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j.18c5580b4d4d15e8 namespace: kserve-ci-e2e-test resourceVersion: "26660" uid: de44dce4-5c08-46fb-8b36-2c46c4102a5c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: 1596af97-aee1-48e6-bd5d-683b67ad2864 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:12Z" name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j.18c5580b4ea13678 namespace: kserve-ci-e2e-test resourceVersion: "26662" uid: 90388cf3-4be9-479e-9283-f1712a946088 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: 1596af97-aee1-48e6-bd5d-683b67ad2864 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:12Z" name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j.18c5580b51e66164 namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: 1393da6a-2c5c-486c-970a-50f19d77c55f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: 1596af97-aee1-48e6-bd5d-683b67ad2864 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:12Z" name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j.18c5580b52591db8 namespace: kserve-ci-e2e-test resourceVersion: "26664" uid: d67b3b73-f660-417b-8ff4-0de42afc89d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: 1596af97-aee1-48e6-bd5d-683b67ad2864 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:12Z" name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j.18c5580b526437db namespace: kserve-ci-e2e-test resourceVersion: "26665" uid: 237befa7-4ca1-4e37-918d-734ecb6ae5b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: 1596af97-aee1-48e6-bd5d-683b67ad2864 kind: Event lastTimestamp: "2026-07-24T21:47:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:13Z" name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j.18c5580b5632b91b namespace: kserve-ci-e2e-test resourceVersion: "26669" uid: d6079c82-e014-4dd6-8cfd-97a58e71a67d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: 1596af97-aee1-48e6-bd5d-683b67ad2864 kind: Event lastTimestamp: "2026-07-24T21:47:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:13Z" name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j.18c5580b56a6cc0b namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: 826c110a-a47c-4f17-a4e0-783de3ef3ff9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-24T21:47:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: 1596af97-aee1-48e6-bd5d-683b67ad2864 kind: Event lastTimestamp: "2026-07-24T21:56:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:47:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:56:29Z" name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j.18c5580b84c8c208 namespace: kserve-ci-e2e-test resourceVersion: "30783" uid: 7aa669ef-7df1-40f0-8444-9d59118dcef5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:56:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: 1596af97-aee1-48e6-bd5d-683b67ad2864 kind: Event lastTimestamp: "2026-07-24T21:56:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:56:27Z" name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j.18c5588c648e436c namespace: kserve-ci-e2e-test resourceVersion: "30744" uid: b2b966f0-be0b-417d-acd9-b99c650d5182 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:56:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: 1596af97-aee1-48e6-bd5d-683b67ad2864 kind: Event lastTimestamp: "2026-07-24T21:56:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:56:27Z" name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j.18c5588c648ed102 namespace: kserve-ci-e2e-test resourceVersion: "30750" uid: 69956314-8d10-4474-94d4-f0deaf58c73a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:56:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: 1596af97-aee1-48e6-bd5d-683b67ad2864 kind: Event lastTimestamp: "2026-07-24T21:56:29Z" message: 'Readiness probe failed: Get "https://10.133.0.42:8643/healthz": dial tcp 10.133.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:56:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:56:29Z" name: success-200-isvc-bce0e-predictor-9867f5768-ttq7j.18c5588cf92b7021 namespace: kserve-ci-e2e-test resourceVersion: "30782" uid: 58de1354-35f4-41f3-a621-f345d7ed2a04 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bce0e-predictor-9867f5768 namespace: kserve-ci-e2e-test resourceVersion: "26602" uid: fa23ecb6-0d83-49d6-b7ce-326172edfdaf kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: 'Created pod: success-200-isvc-bce0e-predictor-9867f5768-ttq7j' metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:12Z" name: success-200-isvc-bce0e-predictor-9867f5768.18c5580b32e9cd69 namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: d4341d82-85b3-4ef0-9639-da2f19a4bbff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bce0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26598" uid: 82eedbc2-7a1e-4e1e-99bf-c2ad1365ac14 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: Scaled up replica set success-200-isvc-bce0e-predictor-9867f5768 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:12Z" name: success-200-isvc-bce0e-predictor.18c5580b314ce0cf namespace: kserve-ci-e2e-test resourceVersion: "26603" uid: 22582cc2-c794-4d04-81e5-17507f5f3dd8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:47:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bce0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26600" uid: 3f3c741b-9d92-4d05-a3ad-02bc84c8d809 kind: Event lastTimestamp: "2026-07-24T21:47:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:42Z" name: success-200-isvc-bce0e-predictor.18c5580eafc5542b namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 1b2db3a8-c921-4dea-bbce-9a9b1ad64a64 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:47:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bce0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26600" uid: 3f3c741b-9d92-4d05-a3ad-02bc84c8d809 kind: Event lastTimestamp: "2026-07-24T21:47:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:42Z" name: success-200-isvc-bce0e-predictor.18c5580eb020adc4 namespace: kserve-ci-e2e-test resourceVersion: "26953" uid: 0f2751c0-9dfa-49f9-adfd-1cfbc03656f9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:47:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bce0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 3f3c741b-9d92-4d05-a3ad-02bc84c8d809 kind: Event lastTimestamp: "2026-07-24T21:48:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:47:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:27Z" name: success-200-isvc-bce0e-predictor.18c55815ad720fb2 namespace: kserve-ci-e2e-test resourceVersion: "27396" uid: 122921f7-f2b3-4154-8c33-ba5fdd32fd06 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:47:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bce0e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 3f3c741b-9d92-4d05-a3ad-02bc84c8d809 kind: Event lastTimestamp: "2026-07-24T21:48:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:47:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:27Z" name: success-200-isvc-bce0e-predictor.18c55815addc8dab namespace: kserve-ci-e2e-test resourceVersion: "27397" uid: 7bef976e-cade-4759-acff-b131ab6abac0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bce0e namespace: kserve-ci-e2e-test resourceVersion: "26574" uid: f5d0015f-c87b-4ca7-8584-b3674a52bba6 kind: Event lastTimestamp: "2026-07-24T21:48:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:48:00Z" name: success-200-isvc-bce0e.18c5580b3152ca45 namespace: kserve-ci-e2e-test resourceVersion: "27143" uid: 4252a6e3-879a-435f-b70b-e20eedfc7355 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bce0e namespace: kserve-ci-e2e-test resourceVersion: "26574" uid: f5d0015f-c87b-4ca7-8584-b3674a52bba6 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-bce0e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bce0e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:47:12Z" name: success-200-isvc-bce0e.18c5580b31f45b0a namespace: kserve-ci-e2e-test resourceVersion: "26608" uid: afc8dfb2-3d08-4ae0-ad78-948755c0be9e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bce0e namespace: kserve-ci-e2e-test resourceVersion: "26574" uid: f5d0015f-c87b-4ca7-8584-b3674a52bba6 kind: Event lastTimestamp: "2026-07-24T21:47:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bce0e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:47:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:47:12Z" name: success-200-isvc-bce0e.18c5580b31f48e24 namespace: kserve-ci-e2e-test resourceVersion: "26609" uid: b2024730-9e20-4cc7-80a2-6654947c6344 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bce0e namespace: kserve-ci-e2e-test resourceVersion: "27141" uid: f5d0015f-c87b-4ca7-8584-b3674a52bba6 kind: Event lastTimestamp: "2026-07-24T21:47:59Z" message: InferenceService [success-200-isvc-bce0e] is Ready metadata: creationTimestamp: "2026-07-24T21:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:47:59Z" name: success-200-isvc-bce0e.18c55816445eaf7f namespace: kserve-ci-e2e-test resourceVersion: "27142" uid: 0765390c-a9a8-486a-95bb-d31d986aede9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:27:11.840760Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3da7-predictor-594869d894-mh5zr namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 34921a6f-8ea9-4879-8925-9240df8703f9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c3da7-predictor-594869d894-mh5zr to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:27:11Z" name: success-200-isvc-c3da7-predictor-594869d894-mh5zr.18c556f3a95d27ba namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: e081f278-de6f-42cf-95bb-e5b86bd52e4e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3da7-predictor-594869d894-mh5zr namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 34921a6f-8ea9-4879-8925-9240df8703f9 kind: Event lastTimestamp: "2026-07-24T21:27:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c3da7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:12Z" name: success-200-isvc-c3da7-predictor-594869d894-mh5zr.18c556f3b44f4927 namespace: kserve-ci-e2e-test resourceVersion: "16418" uid: 878aecb4-c744-4b92-b077-243444709fee reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3da7-predictor-594869d894-mh5zr namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 34921a6f-8ea9-4879-8925-9240df8703f9 kind: Event lastTimestamp: "2026-07-24T21:27:12Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:27:12Z" name: success-200-isvc-c3da7-predictor-594869d894-mh5zr.18c556f3e7013aa5 namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: d39f5a48-8d8d-4520-a4a4-9f5d421927ee reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3da7-predictor-594869d894-mh5zr namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 34921a6f-8ea9-4879-8925-9240df8703f9 kind: Event lastTimestamp: "2026-07-24T21:27:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:12Z" name: success-200-isvc-c3da7-predictor-594869d894-mh5zr.18c556f3e8679054 namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: 6d6456ff-823e-409a-88fd-c7006b16a1dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3da7-predictor-594869d894-mh5zr namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 34921a6f-8ea9-4879-8925-9240df8703f9 kind: Event lastTimestamp: "2026-07-24T21:27:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:12Z" name: success-200-isvc-c3da7-predictor-594869d894-mh5zr.18c556f3eb698833 namespace: kserve-ci-e2e-test resourceVersion: "16438" uid: 80dad14d-5202-4a39-bef1-a35c22785f2c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3da7-predictor-594869d894-mh5zr namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 34921a6f-8ea9-4879-8925-9240df8703f9 kind: Event lastTimestamp: "2026-07-24T21:27:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:12Z" name: success-200-isvc-c3da7-predictor-594869d894-mh5zr.18c556f3ebdcb54b namespace: kserve-ci-e2e-test resourceVersion: "16439" uid: 24a8f6f0-1e71-4fe2-ab30-1dadbadcd559 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3da7-predictor-594869d894-mh5zr namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 34921a6f-8ea9-4879-8925-9240df8703f9 kind: Event lastTimestamp: "2026-07-24T21:27:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:12Z" name: success-200-isvc-c3da7-predictor-594869d894-mh5zr.18c556f3ebe5ba54 namespace: kserve-ci-e2e-test resourceVersion: "16440" uid: b492cebf-a41a-4178-beed-ca38f576a9f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3da7-predictor-594869d894-mh5zr namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 34921a6f-8ea9-4879-8925-9240df8703f9 kind: Event lastTimestamp: "2026-07-24T21:27:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:13Z" name: success-200-isvc-c3da7-predictor-594869d894-mh5zr.18c556f3eed2266d namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 73a51813-1361-4d82-ab9c-d1560b0b4bea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3da7-predictor-594869d894-mh5zr namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 34921a6f-8ea9-4879-8925-9240df8703f9 kind: Event lastTimestamp: "2026-07-24T21:27:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:13Z" name: success-200-isvc-c3da7-predictor-594869d894-mh5zr.18c556f3ef4652fc namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 5e51c0b4-83e4-44f0-be55-9579e1289047 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-24T21:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3da7-predictor-594869d894-mh5zr namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 34921a6f-8ea9-4879-8925-9240df8703f9 kind: Event lastTimestamp: "2026-07-24T21:36:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:29Z" name: success-200-isvc-c3da7-predictor-594869d894-mh5zr.18c556f403053194 namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: 16ff8bdc-8ff2-4089-8086-d0d382bf29c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3da7-predictor-594869d894-mh5zr namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 34921a6f-8ea9-4879-8925-9240df8703f9 kind: Event lastTimestamp: "2026-07-24T21:36:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:26Z" name: success-200-isvc-c3da7-predictor-594869d894-mh5zr.18c55774e10d8ab5 namespace: kserve-ci-e2e-test resourceVersion: "20680" uid: cbb05c1e-25b0-4afd-8025-6d0334663bdc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3da7-predictor-594869d894-mh5zr namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 34921a6f-8ea9-4879-8925-9240df8703f9 kind: Event lastTimestamp: "2026-07-24T21:36:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:26Z" name: success-200-isvc-c3da7-predictor-594869d894-mh5zr.18c55774e10e0e61 namespace: kserve-ci-e2e-test resourceVersion: "20681" uid: 2dea911c-c727-4e5b-8515-dfc2bae82744 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3da7-predictor-594869d894-mh5zr namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 34921a6f-8ea9-4879-8925-9240df8703f9 kind: Event lastTimestamp: "2026-07-24T21:36:29Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:29Z" name: success-200-isvc-c3da7-predictor-594869d894-mh5zr.18c55775777b147c namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: 38f83afe-1633-4b46-a979-1e0b5037e686 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c3da7-predictor-594869d894 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 2a6dedb4-1519-4cb5-85a1-0af03f433d9a kind: Event lastTimestamp: "2026-07-24T21:27:11Z" message: 'Created pod: success-200-isvc-c3da7-predictor-594869d894-mh5zr' metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:11Z" name: success-200-isvc-c3da7-predictor-594869d894.18c556f3a8e3d2bf namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: f03d99e6-4052-48a9-b141-17436bc86413 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c3da7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16342" uid: ed5a1a03-1ded-489a-b632-c7642cf49fde kind: Event lastTimestamp: "2026-07-24T21:27:11Z" message: Scaled up replica set success-200-isvc-c3da7-predictor-594869d894 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:11Z" name: success-200-isvc-c3da7-predictor.18c556f3a73ba5a8 namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: e4af781d-40e7-4365-bafe-a4f72c3c3ab3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3da7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16346" uid: 2122bdb6-1bc9-403c-b9f1-48a84cca72b6 kind: Event lastTimestamp: "2026-07-24T21:27:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:41Z" name: success-200-isvc-c3da7-predictor.18c556f725a478c3 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: cde61ea3-3af9-4c77-8a22-3acb096c7101 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3da7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16346" uid: 2122bdb6-1bc9-403c-b9f1-48a84cca72b6 kind: Event lastTimestamp: "2026-07-24T21:27:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:41Z" name: success-200-isvc-c3da7-predictor.18c556f72610b2a7 namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 8ef44c3a-5852-4878-aa33-5c3f3726efc6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:27:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3da7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16619" uid: 2122bdb6-1bc9-403c-b9f1-48a84cca72b6 kind: Event lastTimestamp: "2026-07-24T21:28:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:28:26Z" name: success-200-isvc-c3da7-predictor.18c556fe23757170 namespace: kserve-ci-e2e-test resourceVersion: "17136" uid: 66f52774-11a4-4925-b672-b3ff7e616a04 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:27:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3da7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16619" uid: 2122bdb6-1bc9-403c-b9f1-48a84cca72b6 kind: Event lastTimestamp: "2026-07-24T21:28:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:28:26Z" name: success-200-isvc-c3da7-predictor.18c556fe23dc429d namespace: kserve-ci-e2e-test resourceVersion: "17137" uid: 8a946a3b-a206-4124-a096-2bc41b55c9ae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:27:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3da7 namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: 7474eb03-0244-4a84-882f-8231b2f550ec kind: Event lastTimestamp: "2026-07-24T21:27:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:27:59Z" name: success-200-isvc-c3da7.18c556f3a7429d6c namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: 3be7cd5e-c83b-45a5-81a5-d11e19c2ca81 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3da7 namespace: kserve-ci-e2e-test resourceVersion: "16884" uid: 7474eb03-0244-4a84-882f-8231b2f550ec kind: Event lastTimestamp: "2026-07-24T21:27:59Z" message: InferenceService [success-200-isvc-c3da7] is Ready metadata: creationTimestamp: "2026-07-24T21:27:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:27:59Z" name: success-200-isvc-c3da7.18c556febe38ad5c namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: b5334333-9a7b-40b6-87c4-7b83c000eaba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:26:21.880082Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15724" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-eea53-predictor-8566895f48-bpxhv to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:26:21Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c556e80779aeba namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: caacc7a5-b2e0-4429-83cc-99757313968c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: "2026-07-24T21:26:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-eea53-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:22Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c556e81411ecec namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: e210c0ad-85a8-4a17-a00f-bf9d56eb6634 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: "2026-07-24T21:26:22Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:26:22Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c556e845f72b99 namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: a2661b61-5b60-4a79-8f4d-c190b90fba44 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: "2026-07-24T21:26:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:22Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c556e8474017e7 namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 15fc3dc4-cc6d-4ee9-8950-a70c3e626f94 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: "2026-07-24T21:26:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:26:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:23Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c556e84a3d77eb namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: ab925006-b3ef-440b-98c6-373eaf1ef575 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: "2026-07-24T21:26:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:26:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:23Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c556e84ab28cc9 namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 48d5d975-f7d9-43e5-8c38-330e3e9abe3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: "2026-07-24T21:26:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:26:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:23Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c556e84abbcff4 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: f777cfac-8080-4e79-8fa2-fb26bda4d208 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: "2026-07-24T21:26:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:26:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:23Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c556e84e906d6d namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: e79b5b4e-5084-4d1d-8efe-139d5a1aef57 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: "2026-07-24T21:26:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:26:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:23Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c556e84f23daf6 namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: a366859f-1c24-425c-81cd-ceef994d24e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:26:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: "2026-07-24T21:27:00Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:26:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:00Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c556e88b5eb411 namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: b9616878-38b6-411a-945f-a0c519705a70 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: "2026-07-24T21:35:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:36Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c5576938c9cbcb namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: 9614c0be-3f7b-40d0-a8fc-033402b8c671 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: "2026-07-24T21:35:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:36Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c5576938caad29 namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 4de9bee5-219c-4423-9a8e-bd4f4fd8f334 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: "2026-07-24T21:35:41Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-24T21:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:41Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c5576a3b86e78a namespace: kserve-ci-e2e-test resourceVersion: "20252" uid: b84a5e98-024c-4e18-b25f-d73b38dfbc40 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eea53-predictor-8566895f48-bpxhv namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 9a287bba-7e81-4f6b-b3db-2b004c7ff3a8 kind: Event lastTimestamp: "2026-07-24T21:35:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: i/o timeout' metadata: creationTimestamp: "2026-07-24T21:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:41Z" name: success-200-isvc-eea53-predictor-8566895f48-bpxhv.18c5576a3bcba658 namespace: kserve-ci-e2e-test resourceVersion: "20253" uid: 893275d5-679b-4415-829e-c6c6f37ae1a9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-eea53-predictor-8566895f48 namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: c9dcc150-bebc-42e4-812b-fde30d14c0ba kind: Event lastTimestamp: "2026-07-24T21:26:21Z" message: 'Created pod: success-200-isvc-eea53-predictor-8566895f48-bpxhv' metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:21Z" name: success-200-isvc-eea53-predictor-8566895f48.18c556e806f5ed75 namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: c34b54f4-20cb-472a-939f-d40d435714ad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-eea53-predictor namespace: kserve-ci-e2e-test resourceVersion: "15700" uid: 1d38b4d0-9226-4ebb-8e28-82c7263e24df kind: Event lastTimestamp: "2026-07-24T21:26:21Z" message: Scaled up replica set success-200-isvc-eea53-predictor-8566895f48 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:21Z" name: success-200-isvc-eea53-predictor.18c556e804546e8a namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: e1bde3a3-7a59-4d40-a3d0-548f1da9629e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:26:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eea53-predictor namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: 8ca3313e-75f4-44a0-938e-e203844251a0 kind: Event lastTimestamp: "2026-07-24T21:26:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:51Z" name: success-200-isvc-eea53-predictor.18c556eb830fd0b8 namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 29f7e0c0-6390-456f-bb3c-bc7bfe3a9ffe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:26:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eea53-predictor namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: 8ca3313e-75f4-44a0-938e-e203844251a0 kind: Event lastTimestamp: "2026-07-24T21:26:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:51Z" name: success-200-isvc-eea53-predictor.18c556eb836a2799 namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: 38b3738e-d0f0-423a-8552-c3b4db866a0c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:27:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eea53-predictor namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 8ca3313e-75f4-44a0-938e-e203844251a0 kind: Event lastTimestamp: "2026-07-24T21:27:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:27:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:36Z" name: success-200-isvc-eea53-predictor.18c556f28096f1c6 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 6cd02e6b-cc7b-40fe-b49f-6bb41ef71f79 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-24T21:27:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eea53-predictor namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 8ca3313e-75f4-44a0-938e-e203844251a0 kind: Event lastTimestamp: "2026-07-24T21:27:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:27:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:36Z" name: success-200-isvc-eea53-predictor.18c556f28112b986 namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: 0c441512-6396-498f-aee0-a20d3776f2fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:26:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-eea53 namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: 4dd08efc-110f-465e-aaa3-0a9a091029e9 kind: Event lastTimestamp: "2026-07-24T21:27:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:27:10Z" name: success-200-isvc-eea53.18c556e8050e14ba namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: b58a5750-68f0-4cc4-88bb-a9cb6b157eb1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-eea53 namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 4dd08efc-110f-465e-aaa3-0a9a091029e9 kind: Event lastTimestamp: "2026-07-24T21:27:10Z" message: InferenceService [success-200-isvc-eea53] is Ready metadata: creationTimestamp: "2026-07-24T21:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:27:10Z" name: success-200-isvc-eea53.18c556f34695dff8 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: e32f9816-aa3b-42f9-a328-a56356445fad reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:36:26.864593Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f761f-predictor-588776b94-kkrwc namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: e697e3ca-f04e-4c0a-8ba9-8330e0b2990b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f761f-predictor-588776b94-kkrwc to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:36:26Z" name: success-200-isvc-f761f-predictor-588776b94-kkrwc.18c55774e35cc595 namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: db301fda-3517-4604-b67a-bbb4b2cfc537 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f761f-predictor-588776b94-kkrwc namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: e697e3ca-f04e-4c0a-8ba9-8330e0b2990b kind: Event lastTimestamp: "2026-07-24T21:36:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f761f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:27Z" name: success-200-isvc-f761f-predictor-588776b94-kkrwc.18c55774eeb33ca9 namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: d61c44e1-bcbf-4c1f-bf03-29f78c2c340c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f761f-predictor-588776b94-kkrwc namespace: kserve-ci-e2e-test resourceVersion: "20693" uid: e697e3ca-f04e-4c0a-8ba9-8330e0b2990b kind: Event lastTimestamp: "2026-07-24T21:36:27Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:36:27Z" name: success-200-isvc-f761f-predictor-588776b94-kkrwc.18c5577520ed6544 namespace: kserve-ci-e2e-test resourceVersion: "20740" uid: c15857eb-61db-47e3-85a3-4ff00f87d754 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f761f-predictor-588776b94-kkrwc namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: e697e3ca-f04e-4c0a-8ba9-8330e0b2990b kind: Event lastTimestamp: "2026-07-24T21:36:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1716" already present on machine metadata: creationTimestamp: "2026-07-24T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:27Z" name: success-200-isvc-f761f-predictor-588776b94-kkrwc.18c55775226b63de namespace: kserve-ci-e2e-test resourceVersion: "20742" uid: 6db81db1-c5af-4bdd-ac19-c209493659a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f761f-predictor-588776b94-kkrwc namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: e697e3ca-f04e-4c0a-8ba9-8330e0b2990b kind: Event lastTimestamp: "2026-07-24T21:36:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-24T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:27Z" name: success-200-isvc-f761f-predictor-588776b94-kkrwc.18c5577525bb4fee namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: f54478a7-ffa6-4e5f-8f13-915c2caec991 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f761f-predictor-588776b94-kkrwc namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: e697e3ca-f04e-4c0a-8ba9-8330e0b2990b kind: Event lastTimestamp: "2026-07-24T21:36:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-24T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:27Z" name: success-200-isvc-f761f-predictor-588776b94-kkrwc.18c557752635ec71 namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: 27c03ad9-4801-4131-b4cf-12e8368dc958 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f761f-predictor-588776b94-kkrwc namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: e697e3ca-f04e-4c0a-8ba9-8330e0b2990b kind: Event lastTimestamp: "2026-07-24T21:36:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-24T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:27Z" name: success-200-isvc-f761f-predictor-588776b94-kkrwc.18c55775263fc5b9 namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 178711b2-b220-41cf-99aa-d9f85b2b7f42 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f761f-predictor-588776b94-kkrwc namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: e697e3ca-f04e-4c0a-8ba9-8330e0b2990b kind: Event lastTimestamp: "2026-07-24T21:36:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-24T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:28Z" name: success-200-isvc-f761f-predictor-588776b94-kkrwc.18c557752aa246c9 namespace: kserve-ci-e2e-test resourceVersion: "20749" uid: 34c6fbb5-dd93-4316-ba11-6ff8fd63d2d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f761f-predictor-588776b94-kkrwc namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: e697e3ca-f04e-4c0a-8ba9-8330e0b2990b kind: Event lastTimestamp: "2026-07-24T21:36:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:28Z" name: success-200-isvc-f761f-predictor-588776b94-kkrwc.18c557752b1764a3 namespace: kserve-ci-e2e-test resourceVersion: "20750" uid: 6af785a9-4eb2-44ef-aeb2-be1db7e3cf01 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f761f-predictor-588776b94-kkrwc namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: e697e3ca-f04e-4c0a-8ba9-8330e0b2990b kind: Event lastTimestamp: "2026-07-24T21:37:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:04Z" name: success-200-isvc-f761f-predictor-588776b94-kkrwc.18c557755a2fe420 namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 7d42d3c0-3371-45a0-b937-978d229e9523 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f761f-predictor-588776b94-kkrwc namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: e697e3ca-f04e-4c0a-8ba9-8330e0b2990b kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-f761f-predictor-588776b94-kkrwc.18c55785391961da namespace: kserve-ci-e2e-test resourceVersion: "21619" uid: daadac4a-5ecf-4d00-9c86-d3604ff7bafd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f761f-predictor-588776b94-kkrwc namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: e697e3ca-f04e-4c0a-8ba9-8330e0b2990b kind: Event lastTimestamp: "2026-07-24T21:37:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-24T21:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:37Z" name: success-200-isvc-f761f-predictor-588776b94-kkrwc.18c557853919f898 namespace: kserve-ci-e2e-test resourceVersion: "21623" uid: e80e6f27-cf81-463a-a78d-ae8505edcacd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f761f-predictor-588776b94-kkrwc namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: e697e3ca-f04e-4c0a-8ba9-8330e0b2990b kind: Event lastTimestamp: "2026-07-24T21:37:39Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-24T21:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:37:39Z" name: success-200-isvc-f761f-predictor-588776b94-kkrwc.18c55785e275d976 namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: 6dd223cf-223d-4423-a4b7-265ed33e4db2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f761f-predictor-588776b94 namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 06975fcd-39c0-417d-aa4f-f30ecad540b7 kind: Event lastTimestamp: "2026-07-24T21:36:26Z" message: 'Created pod: success-200-isvc-f761f-predictor-588776b94-kkrwc' metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:26Z" name: success-200-isvc-f761f-predictor-588776b94.18c55774e27591c6 namespace: kserve-ci-e2e-test resourceVersion: "20686" uid: 67610988-c48f-467e-93a9-1211c07b04bd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f761f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20665" uid: b479b974-b27b-420b-9ee1-060031345170 kind: Event lastTimestamp: "2026-07-24T21:36:26Z" message: Scaled up replica set success-200-isvc-f761f-predictor-588776b94 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:26Z" name: success-200-isvc-f761f-predictor.18c55774df68fbf4 namespace: kserve-ci-e2e-test resourceVersion: "20675" uid: f9a3677f-4c4b-4eaf-8e0d-4088d0d06aee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:36:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f761f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20670" uid: 505dff5b-8022-4cbd-acf3-1a52aa744cd2 kind: Event lastTimestamp: "2026-07-24T21:36:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:56Z" name: success-200-isvc-f761f-predictor.18c557785e145b30 namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: 9dcf8962-dabf-4646-bbe9-1df59f6ebd6e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:36:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f761f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20670" uid: 505dff5b-8022-4cbd-acf3-1a52aa744cd2 kind: Event lastTimestamp: "2026-07-24T21:36:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:36:56Z" name: success-200-isvc-f761f-predictor.18c557785e847eab namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: d5693d97-181f-40f1-a4fe-edd378f2b3a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:37:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f761f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: 505dff5b-8022-4cbd-acf3-1a52aa744cd2 kind: Event lastTimestamp: "2026-07-24T21:37:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:26Z" name: success-200-isvc-f761f-predictor.18c5577f5ce123f1 namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: a9c00d57-fc0e-4d95-919d-36dc51fa5fb3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:37:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f761f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: 505dff5b-8022-4cbd-acf3-1a52aa744cd2 kind: Event lastTimestamp: "2026-07-24T21:37:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-24T21:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:37:26Z" name: success-200-isvc-f761f-predictor.18c5577f5dee8993 namespace: kserve-ci-e2e-test resourceVersion: "21478" uid: 46bf51a1-63c9-436a-a060-dec4109a26b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f761f namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 7dd2b2a5-af3a-49e9-a0ae-3ef744c72599 kind: Event lastTimestamp: "2026-07-24T21:37:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:14Z" name: success-200-isvc-f761f.18c55774df80728c namespace: kserve-ci-e2e-test resourceVersion: "21349" uid: ec64ca47-dc84-4920-aae3-19dce2d6c2df reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f761f namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 7dd2b2a5-af3a-49e9-a0ae-3ef744c72599 kind: Event lastTimestamp: "2026-07-24T21:36:26Z" message: 'Failed to update status for InferenceService "success-200-isvc-f761f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f761f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:36:26Z" name: success-200-isvc-f761f.18c55774e04ae77e namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: 661f349b-8673-4765-8aa6-911b5b134831 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:36:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f761f namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 7dd2b2a5-af3a-49e9-a0ae-3ef744c72599 kind: Event lastTimestamp: "2026-07-24T21:36:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f761f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-24T21:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:36:26Z" name: success-200-isvc-f761f.18c55774e04b35da namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: beb2350e-add6-4a0d-8ff5-b28066988027 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f761f namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: 7dd2b2a5-af3a-49e9-a0ae-3ef744c72599 kind: Event lastTimestamp: "2026-07-24T21:37:14Z" message: InferenceService [success-200-isvc-f761f] is Ready metadata: creationTimestamp: "2026-07-24T21:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:37:14Z" name: success-200-isvc-f761f.18c5578016114b09 namespace: kserve-ci-e2e-test resourceVersion: "21348" uid: eacff0fa-4dd7-42c5-a2d3-87f46e93b237 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:26:11.502343Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-77ecc-597566fc6b-4bnvg namespace: kserve-ci-e2e-test resourceVersion: "15558" uid: 9d76e551-a8c7-43fa-81c0-a176fee9474a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-77ecc-597566fc6b-4bnvg to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:26:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:26:11Z" name: switch-graph-77ecc-597566fc6b-4bnvg.18c556e59ce9f5b2 namespace: kserve-ci-e2e-test resourceVersion: "15563" uid: f324863b-2ea0-4e73-ab23-0ab9e6256d1a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-77ecc-597566fc6b-4bnvg namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 9d76e551-a8c7-43fa-81c0-a176fee9474a kind: Event lastTimestamp: "2026-07-24T21:26:11Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:26:11Z" name: switch-graph-77ecc-597566fc6b-4bnvg.18c556e5b6653b0e namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: 4f1d8973-1894-4acb-8fa5-5569ebc1f88a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-77ecc} kind: Pod name: switch-graph-77ecc-597566fc6b-4bnvg namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: 9d76e551-a8c7-43fa-81c0-a176fee9474a kind: Event lastTimestamp: "2026-07-24T21:26:11Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:d388f8d2997cdb455a73f33beea2e1aa8fe959b1d5682a0690a87a2319ba2789" metadata: creationTimestamp: "2026-07-24T21:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:11Z" name: switch-graph-77ecc-597566fc6b-4bnvg.18c556e5b7c8e386 namespace: kserve-ci-e2e-test resourceVersion: "15581" uid: 8ade7660-cd52-49d0-b63a-778d7688a7cb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-77ecc} kind: Pod name: switch-graph-77ecc-597566fc6b-4bnvg namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: 9d76e551-a8c7-43fa-81c0-a176fee9474a kind: Event lastTimestamp: "2026-07-24T21:26:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:d388f8d2997cdb455a73f33beea2e1aa8fe959b1d5682a0690a87a2319ba2789" in 2.086s (2.086s including waiting). Image size: 218233959 bytes.' metadata: creationTimestamp: "2026-07-24T21:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:14Z" name: switch-graph-77ecc-597566fc6b-4bnvg.18c556e63427540d namespace: kserve-ci-e2e-test resourceVersion: "15596" uid: 15697aba-0c07-426d-8e96-2e403f888481 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-77ecc} kind: Pod name: switch-graph-77ecc-597566fc6b-4bnvg namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: 9d76e551-a8c7-43fa-81c0-a176fee9474a kind: Event lastTimestamp: "2026-07-24T21:26:14Z" message: 'Created container: switch-graph-77ecc' metadata: creationTimestamp: "2026-07-24T21:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:14Z" name: switch-graph-77ecc-597566fc6b-4bnvg.18c556e637515620 namespace: kserve-ci-e2e-test resourceVersion: "15597" uid: d5c45e8f-8e29-41ff-b8b1-06fb03141a68 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-77ecc} kind: Pod name: switch-graph-77ecc-597566fc6b-4bnvg namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: 9d76e551-a8c7-43fa-81c0-a176fee9474a kind: Event lastTimestamp: "2026-07-24T21:26:14Z" message: Started container switch-graph-77ecc metadata: creationTimestamp: "2026-07-24T21:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:14Z" name: switch-graph-77ecc-597566fc6b-4bnvg.18c556e637cc5d3e namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: 170ac470-323d-48bb-87d3-0d4d475d0f61 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-77ecc} kind: Pod name: switch-graph-77ecc-597566fc6b-4bnvg namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: 9d76e551-a8c7-43fa-81c0-a176fee9474a kind: Event lastTimestamp: "2026-07-24T21:26:21Z" message: Stopping container switch-graph-77ecc metadata: creationTimestamp: "2026-07-24T21:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:21Z" name: switch-graph-77ecc-597566fc6b-4bnvg.18c556e7fbc40f97 namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: 57c80225-95dd-4e9b-afd6-2108ebe62c2d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:26:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-77ecc} kind: Pod name: switch-graph-77ecc-597566fc6b-4bnvg namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: 9d76e551-a8c7-43fa-81c0-a176fee9474a kind: Event lastTimestamp: "2026-07-24T21:26:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-24T21:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:26:50Z" name: switch-graph-77ecc-597566fc6b-4bnvg.18c556e8e78deff8 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 450dfccc-7879-42f9-b8ef-9940e1738934 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-77ecc-597566fc6b namespace: kserve-ci-e2e-test resourceVersion: "15555" uid: 48429f22-44ac-4385-8482-ccbc6ff5f8e5 kind: Event lastTimestamp: "2026-07-24T21:26:11Z" message: 'Created pod: switch-graph-77ecc-597566fc6b-4bnvg' metadata: creationTimestamp: "2026-07-24T21:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:11Z" name: switch-graph-77ecc-597566fc6b.18c556e59c71f849 namespace: kserve-ci-e2e-test resourceVersion: "15559" uid: cc9ddc69-378f-4f26-999f-157ad8f7c800 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-77ecc namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: 93383bcb-cd03-4411-ab39-84d036a2b4b7 kind: Event lastTimestamp: "2026-07-24T21:26:11Z" message: Scaled up replica set switch-graph-77ecc-597566fc6b from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:26:11Z" name: switch-graph-77ecc.18c556e59b816b4c namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 54dfde53-12d5-438a-88ef-3c783409c0fa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:26:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-77ecc namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: 372c6078-cb22-4579-8b7a-dac4f73b55bb kind: Event lastTimestamp: "2026-07-24T21:26:11Z" message: InferenceGraph [switch-graph-77ecc] is Ready metadata: creationTimestamp: "2026-07-24T21:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:26:11Z" name: switch-graph-77ecc.18c556e5a14748de namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: 9c85498c-f3e6-40ab-979a-fe665ad2ba85 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:47:52.089997Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a916f-6d9bcff485-5rs4f namespace: kserve-ci-e2e-test resourceVersion: "27039" uid: cd9e0c35-68db-42f7-a4a9-066869594ece kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-a916f-6d9bcff485-5rs4f to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:47:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:47:52Z" name: switch-graph-a916f-6d9bcff485-5rs4f.18c558146dfaac52 namespace: kserve-ci-e2e-test resourceVersion: "27046" uid: b73960f5-bf84-4bf9-94e4-9426d1c0f94d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:47:52Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a916f-6d9bcff485-5rs4f namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: cd9e0c35-68db-42f7-a4a9-066869594ece kind: Event lastTimestamp: "2026-07-24T21:47:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-a916f-serving-cert" not found' metadata: creationTimestamp: "2026-07-24T21:47:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:52Z" name: switch-graph-a916f-6d9bcff485-5rs4f.18c558147d1dd4f5 namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: 3eb6c5fb-e3c9-4ca5-8eaa-29a8758bccdc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:54Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a916f-6d9bcff485-5rs4f namespace: kserve-ci-e2e-test resourceVersion: "27050" uid: cd9e0c35-68db-42f7-a4a9-066869594ece kind: Event lastTimestamp: "2026-07-24T21:47:54Z" message: Add eth0 [10.132.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:47:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:47:54Z" name: switch-graph-a916f-6d9bcff485-5rs4f.18c55814e14da4e0 namespace: kserve-ci-e2e-test resourceVersion: "27074" uid: 58cb734e-e978-4731-a37d-4fa2e961a913 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a916f} kind: Pod name: switch-graph-a916f-6d9bcff485-5rs4f namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: cd9e0c35-68db-42f7-a4a9-066869594ece kind: Event lastTimestamp: "2026-07-24T21:47:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d388f8d2997cdb455a73f33beea2e1aa8fe959b1d5682a0690a87a2319ba2789" already present on machine metadata: creationTimestamp: "2026-07-24T21:47:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:54Z" name: switch-graph-a916f-6d9bcff485-5rs4f.18c55814e2817e78 namespace: kserve-ci-e2e-test resourceVersion: "27076" uid: 00eee7ad-8732-43e1-85bd-02dc8ddbf48e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a916f} kind: Pod name: switch-graph-a916f-6d9bcff485-5rs4f namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: cd9e0c35-68db-42f7-a4a9-066869594ece kind: Event lastTimestamp: "2026-07-24T21:47:54Z" message: 'Created container: switch-graph-a916f' metadata: creationTimestamp: "2026-07-24T21:47:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:54Z" name: switch-graph-a916f-6d9bcff485-5rs4f.18c55814e61a3cdd namespace: kserve-ci-e2e-test resourceVersion: "27079" uid: 580c5c10-3ed2-4d8f-946d-4687d09868c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a916f} kind: Pod name: switch-graph-a916f-6d9bcff485-5rs4f namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: cd9e0c35-68db-42f7-a4a9-066869594ece kind: Event lastTimestamp: "2026-07-24T21:47:54Z" message: Started container switch-graph-a916f metadata: creationTimestamp: "2026-07-24T21:47:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:47:54Z" name: switch-graph-a916f-6d9bcff485-5rs4f.18c55814e6ab0372 namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: 8c4b891d-bff8-43ce-8727-0b462792c4af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T22:04:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a916f} kind: Pod name: switch-graph-a916f-6d9bcff485-5rs4f namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: cd9e0c35-68db-42f7-a4a9-066869594ece kind: Event lastTimestamp: "2026-07-24T22:04:11Z" message: Stopping container switch-graph-a916f metadata: creationTimestamp: "2026-07-24T22:04:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T22:04:11Z" name: switch-graph-a916f-6d9bcff485-5rs4f.18c558f86d60dab3 namespace: kserve-ci-e2e-test resourceVersion: "33941" uid: 79e7def5-14aa-4e1d-a40c-e92a4e25b5c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-24T22:04:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a916f} kind: Pod name: switch-graph-a916f-6d9bcff485-5rs4f namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: cd9e0c35-68db-42f7-a4a9-066869594ece kind: Event lastTimestamp: "2026-07-24T22:04:35Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-24T22:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T22:04:35Z" name: switch-graph-a916f-6d9bcff485-5rs4f.18c558f977d4eea4 namespace: kserve-ci-e2e-test resourceVersion: "34191" uid: 480d661f-33db-4480-86d7-7cfd0b54de92 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-a916f-6d9bcff485 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: e92feb86-0fd1-4463-9a24-30183149e7e9 kind: Event lastTimestamp: "2026-07-24T21:47:52Z" message: 'Created pod: switch-graph-a916f-6d9bcff485-5rs4f' metadata: creationTimestamp: "2026-07-24T21:47:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:52Z" name: switch-graph-a916f-6d9bcff485.18c558146d94ca87 namespace: kserve-ci-e2e-test resourceVersion: "27041" uid: 85d6249a-db6f-48f2-95ae-6e8fe01828fa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-a916f namespace: kserve-ci-e2e-test resourceVersion: "27033" uid: 996445ef-65a0-4746-89f3-15b3cac3d0ed kind: Event lastTimestamp: "2026-07-24T21:47:52Z" message: Scaled up replica set switch-graph-a916f-6d9bcff485 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:47:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:47:52Z" name: switch-graph-a916f.18c558146cb6cd88 namespace: kserve-ci-e2e-test resourceVersion: "27036" uid: 38bd72ce-0281-4cbf-aaab-bb0831df7e31 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:47:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-a916f namespace: kserve-ci-e2e-test resourceVersion: "27053" uid: e89817f3-5f9e-4ca1-b395-bd6bb113c5f1 kind: Event lastTimestamp: "2026-07-24T21:47:52Z" message: InferenceGraph [switch-graph-a916f] is Ready metadata: creationTimestamp: "2026-07-24T21:47:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:47:52Z" name: switch-graph-a916f.18c558147241d2f7 namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: 3736ecfe-ff3f-49d8-a727-016cad0b2ec1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:48:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a916f namespace: kserve-ci-e2e-test resourceVersion: "27034" uid: acdd4e8e-c326-4e65-9a80-6348561c156b kind: Event lastTimestamp: "2026-07-24T21:48:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:22Z" name: switch-graph-a916f.18c55817eb21c959 namespace: kserve-ci-e2e-test resourceVersion: "27354" uid: 3461a0a5-a8a1-4841-afb7-24aa6a0e7758 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:48:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a916f namespace: kserve-ci-e2e-test resourceVersion: "27034" uid: acdd4e8e-c326-4e65-9a80-6348561c156b kind: Event lastTimestamp: "2026-07-24T21:48:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:48:22Z" name: switch-graph-a916f.18c55817eb8a407b namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: fc8ea364-7853-432a-a6cf-d3c4349cd24a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-24T21:27:21.923482Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-eea53-7b4cdf96d5-xwlds namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: 7228bc66-4068-48e3-84ff-031be52336cb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-eea53-7b4cdf96d5-xwlds to ip-10-0-140-162.ec2.internal metadata: creationTimestamp: "2026-07-24T21:27:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-24T21:27:21Z" name: switch-graph-eea53-7b4cdf96d5-xwlds.18c556f6025782e8 namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: e74b5126-3d0f-4deb-8781-50dca2667aee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7758b59f87-2j2x9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:22Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-eea53-7b4cdf96d5-xwlds namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 7228bc66-4068-48e3-84ff-031be52336cb kind: Event lastTimestamp: "2026-07-24T21:27:22Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-24T21:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-24T21:27:22Z" name: switch-graph-eea53-7b4cdf96d5-xwlds.18c556f61bf95e96 namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: 4ef1fc9c-f57b-455b-b30e-81f78cddbde9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eea53} kind: Pod name: switch-graph-eea53-7b4cdf96d5-xwlds namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 7228bc66-4068-48e3-84ff-031be52336cb kind: Event lastTimestamp: "2026-07-24T21:27:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:d388f8d2997cdb455a73f33beea2e1aa8fe959b1d5682a0690a87a2319ba2789" already present on machine metadata: creationTimestamp: "2026-07-24T21:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:22Z" name: switch-graph-eea53-7b4cdf96d5-xwlds.18c556f61d41d976 namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: eb32d39a-9737-4f49-9dc7-1e0a16aafb69 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eea53} kind: Pod name: switch-graph-eea53-7b4cdf96d5-xwlds namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 7228bc66-4068-48e3-84ff-031be52336cb kind: Event lastTimestamp: "2026-07-24T21:27:22Z" message: 'Created container: switch-graph-eea53' metadata: creationTimestamp: "2026-07-24T21:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:22Z" name: switch-graph-eea53-7b4cdf96d5-xwlds.18c556f6206a87f5 namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: f8dde1e5-286e-4f82-b670-6967c4691b35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eea53} kind: Pod name: switch-graph-eea53-7b4cdf96d5-xwlds namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 7228bc66-4068-48e3-84ff-031be52336cb kind: Event lastTimestamp: "2026-07-24T21:27:22Z" message: Started container switch-graph-eea53 metadata: creationTimestamp: "2026-07-24T21:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:27:22Z" name: switch-graph-eea53-7b4cdf96d5-xwlds.18c556f620eb1258 namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: 0c268647-67d9-4c3b-81a5-3cac77ea1906 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eea53} kind: Pod name: switch-graph-eea53-7b4cdf96d5-xwlds namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 7228bc66-4068-48e3-84ff-031be52336cb kind: Event lastTimestamp: "2026-07-24T21:35:36Z" message: Stopping container switch-graph-eea53 metadata: creationTimestamp: "2026-07-24T21:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:35:36Z" name: switch-graph-eea53-7b4cdf96d5-xwlds.18c5576930cb743c namespace: kserve-ci-e2e-test resourceVersion: "20082" uid: c197182d-3df2-4698-a777-a464ddab4f75 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-24T21:35:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eea53} kind: Pod name: switch-graph-eea53-7b4cdf96d5-xwlds namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 7228bc66-4068-48e3-84ff-031be52336cb kind: Event lastTimestamp: "2026-07-24T21:36:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-24T21:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-24T21:36:03Z" name: switch-graph-eea53-7b4cdf96d5-xwlds.18c55769bc48cf7e namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 02d277d0-fa61-4480-9e94-19ee567557cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-162.ec2.internal source: component: kubelet host: ip-10-0-140-162.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-eea53-7b4cdf96d5 namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 8f7a7ede-60e4-4b87-bfac-0e62851b5b2f kind: Event lastTimestamp: "2026-07-24T21:27:21Z" message: 'Created pod: switch-graph-eea53-7b4cdf96d5-xwlds' metadata: creationTimestamp: "2026-07-24T21:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:21Z" name: switch-graph-eea53-7b4cdf96d5.18c556f6018fd0ad namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 086ca43f-e254-403a-8be2-cdb6036a5348 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-eea53 namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: fdaa4152-842a-46bf-b384-6d89a92cdb5c kind: Event lastTimestamp: "2026-07-24T21:27:21Z" message: Scaled up replica set switch-graph-eea53-7b4cdf96d5 from 0 to 1 metadata: creationTimestamp: "2026-07-24T21:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:21Z" name: switch-graph-eea53.18c556f600cfae3a namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: c877ad3a-74a6-450c-b474-40f90c9a0a46 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-24T21:27:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-eea53 namespace: kserve-ci-e2e-test resourceVersion: "16565" uid: d7ceb51b-10ed-4cd5-b604-4b319d9c1832 kind: Event lastTimestamp: "2026-07-24T21:27:21Z" message: InferenceGraph [switch-graph-eea53] is Ready metadata: creationTimestamp: "2026-07-24T21:27:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-24T21:27:21Z" name: switch-graph-eea53.18c556f6067f374c namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: 3c861b46-b2ee-4c5d-9d3e-eae46e021d43 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:27:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-eea53 namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: cb76ff3f-3d31-40a6-b441-db7dc4b4fe9a kind: Event lastTimestamp: "2026-07-24T21:27:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:51Z" name: switch-graph-eea53.18c556f97f28b191 namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: 4fe4c6ca-2600-4560-ac53-c69db36121ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-24T21:27:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-eea53 namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: cb76ff3f-3d31-40a6-b441-db7dc4b4fe9a kind: Event lastTimestamp: "2026-07-24T21:27:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-24T21:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-24T21:27:51Z" name: switch-graph-eea53.18c556f97f6006ba namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: ef6f0c08-d4a8-4e77-bfe3-bf31a12733fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34215"