--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:17:12.362502Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-36a90-744c45996f-jphzx namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: a37917e7-eccc-497b-a65d-0c59e8801670 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-36a90-744c45996f-jphzx to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:17:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:17:12Z" name: ensemble-graph-36a90-744c45996f-jphzx.18a6df4ebfcc2908 namespace: kserve-ci-e2e-test resourceVersion: "20456" uid: 0e19c5e0-9c02-49f6-82de-6b6430424391 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:12Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-36a90-744c45996f-jphzx namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: a37917e7-eccc-497b-a65d-0c59e8801670 kind: Event lastTimestamp: "2026-04-16T15:17:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-36a90-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T15:17:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:12Z" name: ensemble-graph-36a90-744c45996f-jphzx.18a6df4ecb45d02b namespace: kserve-ci-e2e-test resourceVersion: "20471" uid: 7848a705-6140-4b19-b887-7ab487f399fe reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:13Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-36a90-744c45996f-jphzx namespace: kserve-ci-e2e-test resourceVersion: "20463" uid: a37917e7-eccc-497b-a65d-0c59e8801670 kind: Event lastTimestamp: "2026-04-16T15:17:13Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:17:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:17:13Z" name: ensemble-graph-36a90-744c45996f-jphzx.18a6df4efd178898 namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 34a7d65d-a946-4808-8755-1982f2c3c10c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-36a90} kind: Pod name: ensemble-graph-36a90-744c45996f-jphzx namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: a37917e7-eccc-497b-a65d-0c59e8801670 kind: Event lastTimestamp: "2026-04-16T15:17:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" already present on machine metadata: creationTimestamp: "2026-04-16T15:17:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:13Z" name: ensemble-graph-36a90-744c45996f-jphzx.18a6df4efe40f9f0 namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: 9e79a84f-488d-4230-8c1c-3c3ff2af0d12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-36a90} kind: Pod name: ensemble-graph-36a90-744c45996f-jphzx namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: a37917e7-eccc-497b-a65d-0c59e8801670 kind: Event lastTimestamp: "2026-04-16T15:17:13Z" message: 'Created container: ensemble-graph-36a90' metadata: creationTimestamp: "2026-04-16T15:17:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:13Z" name: ensemble-graph-36a90-744c45996f-jphzx.18a6df4f0220c51c namespace: kserve-ci-e2e-test resourceVersion: "20482" uid: 4d544386-782b-4de1-a50d-d10b179dc724 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-36a90} kind: Pod name: ensemble-graph-36a90-744c45996f-jphzx namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: a37917e7-eccc-497b-a65d-0c59e8801670 kind: Event lastTimestamp: "2026-04-16T15:17:13Z" message: Started container ensemble-graph-36a90 metadata: creationTimestamp: "2026-04-16T15:17:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:13Z" name: ensemble-graph-36a90-744c45996f-jphzx.18a6df4f028f210a namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: 3e5fa2f7-e975-4064-8250-b89e056c11b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-36a90} kind: Pod name: ensemble-graph-36a90-744c45996f-jphzx namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: a37917e7-eccc-497b-a65d-0c59e8801670 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: Stopping container ensemble-graph-36a90 metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:22Z" name: ensemble-graph-36a90-744c45996f-jphzx.18a6df51184c9d0b namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: fc6bb6e2-e5da-4728-b9dd-40dbc50f0f3e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:17:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-36a90} kind: Pod name: ensemble-graph-36a90-744c45996f-jphzx namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: a37917e7-eccc-497b-a65d-0c59e8801670 kind: Event lastTimestamp: "2026-04-16T15:17:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:49Z" name: ensemble-graph-36a90-744c45996f-jphzx.18a6df51a4fcaa57 namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: e997a7fa-e634-4947-868c-637d4b92aa93 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-36a90-744c45996f namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: c30ae2cb-5f66-43f4-99f0-b00ec13e6576 kind: Event lastTimestamp: "2026-04-16T15:17:12Z" message: 'Created pod: ensemble-graph-36a90-744c45996f-jphzx' metadata: creationTimestamp: "2026-04-16T15:17:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:12Z" name: ensemble-graph-36a90-744c45996f.18a6df4ebf481ac4 namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 3e379eb0-9607-44c0-841d-aa4903513d80 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-36a90 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: 8df375ea-06b4-408c-8eb9-c4c53786eb1f kind: Event lastTimestamp: "2026-04-16T15:17:12Z" message: Scaled up replica set ensemble-graph-36a90-744c45996f from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:17:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:12Z" name: ensemble-graph-36a90.18a6df4ebebccae1 namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 1fb68a63-90eb-4e54-8bcc-bae45508ceb8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-36a90 namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: ce1281ec-38df-4306-a3ca-a6557b9741bb kind: Event lastTimestamp: "2026-04-16T15:17:12Z" message: InferenceGraph [ensemble-graph-36a90] is Ready metadata: creationTimestamp: "2026-04-16T15:17:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:17:12Z" name: ensemble-graph-36a90.18a6df4ec20fc041 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: aa6817d5-52a9-482d-bcba-056d56377a86 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:18:22.636668Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b2be2-6df585b7fc-bqfmb namespace: kserve-ci-e2e-test resourceVersion: "21385" uid: be3951c5-9ed2-4522-be4f-8b9b4466e37b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-b2be2-6df585b7fc-bqfmb to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:18:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:18:22Z" name: ensemble-graph-b2be2-6df585b7fc-bqfmb.18a6df5f1c76cc6e namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: 8f4648b7-9996-4735-bf89-d08e5944b0d7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:23Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b2be2-6df585b7fc-bqfmb namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: be3951c5-9ed2-4522-be4f-8b9b4466e37b kind: Event lastTimestamp: "2026-04-16T15:18:23Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:18:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:18:23Z" name: ensemble-graph-b2be2-6df585b7fc-bqfmb.18a6df5f35e0e6a5 namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 15d9cefd-21c5-4cb9-a1de-30a0b3b11d7c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2be2} kind: Pod name: ensemble-graph-b2be2-6df585b7fc-bqfmb namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: be3951c5-9ed2-4522-be4f-8b9b4466e37b kind: Event lastTimestamp: "2026-04-16T15:18:23Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" already present on machine metadata: creationTimestamp: "2026-04-16T15:18:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:23Z" name: ensemble-graph-b2be2-6df585b7fc-bqfmb.18a6df5f3717a9a7 namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 4e79e04f-9a42-45d2-9e82-3f4564482b64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2be2} kind: Pod name: ensemble-graph-b2be2-6df585b7fc-bqfmb namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: be3951c5-9ed2-4522-be4f-8b9b4466e37b kind: Event lastTimestamp: "2026-04-16T15:18:23Z" message: 'Created container: ensemble-graph-b2be2' metadata: creationTimestamp: "2026-04-16T15:18:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:23Z" name: ensemble-graph-b2be2-6df585b7fc-bqfmb.18a6df5f3a4b20b6 namespace: kserve-ci-e2e-test resourceVersion: "21410" uid: 1dc8cf07-2ca3-429d-b6c5-9ed3beef08c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2be2} kind: Pod name: ensemble-graph-b2be2-6df585b7fc-bqfmb namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: be3951c5-9ed2-4522-be4f-8b9b4466e37b kind: Event lastTimestamp: "2026-04-16T15:18:23Z" message: Started container ensemble-graph-b2be2 metadata: creationTimestamp: "2026-04-16T15:18:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:23Z" name: ensemble-graph-b2be2-6df585b7fc-bqfmb.18a6df5f3ab878b3 namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: 16985325-f46d-4e20-874a-3e9b72d3c198 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2be2} kind: Pod name: ensemble-graph-b2be2-6df585b7fc-bqfmb namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: be3951c5-9ed2-4522-be4f-8b9b4466e37b kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: Stopping container ensemble-graph-b2be2 metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:26:37Z" name: ensemble-graph-b2be2-6df585b7fc-bqfmb.18a6dfd24d0e7b02 namespace: kserve-ci-e2e-test resourceVersion: "25197" uid: 303a752f-5dd7-49d0-8ecf-5d0341bad0b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:26:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2be2} kind: Pod name: ensemble-graph-b2be2-6df585b7fc-bqfmb namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: be3951c5-9ed2-4522-be4f-8b9b4466e37b kind: Event lastTimestamp: "2026-04-16T15:27:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:05Z" name: ensemble-graph-b2be2-6df585b7fc-bqfmb.18a6dfd2ec555dc9 namespace: kserve-ci-e2e-test resourceVersion: "25528" uid: f9f848d2-2624-4a32-bc20-23160614cbc2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-b2be2-6df585b7fc namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 244b6c30-58c0-4bb6-9123-9f26ce3f35f3 kind: Event lastTimestamp: "2026-04-16T15:18:22Z" message: 'Created pod: ensemble-graph-b2be2-6df585b7fc-bqfmb' metadata: creationTimestamp: "2026-04-16T15:18:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:22Z" name: ensemble-graph-b2be2-6df585b7fc.18a6df5f1c136f46 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 7f0625a2-b13e-4057-a182-18f6fee1fc70 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: ed799538-c7fb-4b9c-bb30-8b3f975890db kind: Event lastTimestamp: "2026-04-16T15:18:22Z" message: Scaled up replica set ensemble-graph-b2be2-6df585b7fc from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:18:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:22Z" name: ensemble-graph-b2be2.18a6df5f1b8d1854 namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 3d41f668-1ed9-467a-8d0c-84316b4677c1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 2175b27e-ba90-45e4-8014-16c773d148bc kind: Event lastTimestamp: "2026-04-16T15:18:22Z" message: InferenceGraph [ensemble-graph-b2be2] is Ready metadata: creationTimestamp: "2026-04-16T15:18:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:22Z" name: ensemble-graph-b2be2.18a6df5f1ef1c466 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 8ce3e8cc-4c64-4a93-84ba-a67da6a0f312 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: f3d5af3c-7d46-47c7-95df-50c0cb7e8bff kind: Event lastTimestamp: "2026-04-16T15:18:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:18:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:37Z" name: ensemble-graph-b2be2.18a6df6299eca28e namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: 0cffd3aa-57c2-4688-9527-eb65f6de1a57 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: f3d5af3c-7d46-47c7-95df-50c0cb7e8bff kind: Event lastTimestamp: "2026-04-16T15:18:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:18:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:37Z" name: ensemble-graph-b2be2.18a6df629a434389 namespace: kserve-ci-e2e-test resourceVersion: "21532" uid: 8a2a77d2-02d4-4392-8b46-2469efe636f7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: f3d5af3c-7d46-47c7-95df-50c0cb7e8bff kind: Event lastTimestamp: "2026-04-16T15:18:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:18:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:52Z" name: ensemble-graph-b2be2.18a6df6618d6ee57 namespace: kserve-ci-e2e-test resourceVersion: "21651" uid: e5f3f40b-d81d-46aa-9812-3df151cd9c99 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: f3d5af3c-7d46-47c7-95df-50c0cb7e8bff kind: Event lastTimestamp: "2026-04-16T15:18:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:18:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:52Z" name: ensemble-graph-b2be2.18a6df661913998b namespace: kserve-ci-e2e-test resourceVersion: "21653" uid: c759ed92-968d-4c3a-85dd-569b77752dc2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:26:37.570845Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7 namespace: kserve-ci-e2e-test resourceVersion: "25268" uid: 120e5528-f2aa-4e5f-ba13-209694829f54 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1fc59-predictor-856bfdcfb-97jv7 to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:26:37Z" name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7.18a6dfd258d702f6 namespace: kserve-ci-e2e-test resourceVersion: "25272" uid: 2ecaa7e3-b6ad-4b79-a93b-7d8fc4552e66 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7 namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: 120e5528-f2aa-4e5f-ba13-209694829f54 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: Add eth0 [10.134.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:26:37Z" name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7.18a6dfd2602b4d69 namespace: kserve-ci-e2e-test resourceVersion: "25285" uid: c26c13a0-61b3-42b0-a5cc-3d473d27bc96 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7 namespace: kserve-ci-e2e-test resourceVersion: "25270" uid: 120e5528-f2aa-4e5f-ba13-209694829f54 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:26:37Z" name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7.18a6dfd26146b933 namespace: kserve-ci-e2e-test resourceVersion: "25287" uid: 6d47fe2c-64e3-4707-bb4c-f89fc363f636 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7 namespace: kserve-ci-e2e-test resourceVersion: "25270" uid: 120e5528-f2aa-4e5f-ba13-209694829f54 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:26:37Z" name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7.18a6dfd2651000ae namespace: kserve-ci-e2e-test resourceVersion: "25291" uid: 9dcaab80-605f-450a-ae60-2016d6c720b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7 namespace: kserve-ci-e2e-test resourceVersion: "25270" uid: 120e5528-f2aa-4e5f-ba13-209694829f54 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:26:37Z" name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7.18a6dfd26580735d namespace: kserve-ci-e2e-test resourceVersion: "25292" uid: 0c277a9e-c4b5-450a-89bb-c16c12f6b8c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T15:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7 namespace: kserve-ci-e2e-test resourceVersion: "25270" uid: 120e5528-f2aa-4e5f-ba13-209694829f54 kind: Event lastTimestamp: "2026-04-16T15:27:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.50:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:49Z" name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7.18a6dfd29d75130f namespace: kserve-ci-e2e-test resourceVersion: "26181" uid: dc199dbb-b0f1-4b5f-8bde-143da8a20763 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7 namespace: kserve-ci-e2e-test resourceVersion: "25270" uid: 120e5528-f2aa-4e5f-ba13-209694829f54 kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:47Z" name: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7.18a6dfe2b58b0081 namespace: kserve-ci-e2e-test resourceVersion: "26123" uid: 74f0205b-13db-49c2-94a4-f65028852441 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1fc59-predictor-856bfdcfb namespace: kserve-ci-e2e-test resourceVersion: "25254" uid: 486caa77-1bbd-4cbc-ac4c-c0451c0ae031 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: 'Created pod: error-404-isvc-1fc59-predictor-856bfdcfb-97jv7' metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:26:37Z" name: error-404-isvc-1fc59-predictor-856bfdcfb.18a6dfd2587b11d5 namespace: kserve-ci-e2e-test resourceVersion: "25269" uid: 6fb5f001-7e73-438d-af19-91e0eda952b3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1fc59-predictor namespace: kserve-ci-e2e-test resourceVersion: "25252" uid: 8574dbbd-3c19-4f57-8cfd-c51dc6509d28 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: Scaled up replica set error-404-isvc-1fc59-predictor-856bfdcfb from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:26:37Z" name: error-404-isvc-1fc59-predictor.18a6dfd257041bd7 namespace: kserve-ci-e2e-test resourceVersion: "25258" uid: ae964cc5-474f-4fae-b889-a910f718f998 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1fc59-predictor namespace: kserve-ci-e2e-test resourceVersion: "25255" uid: 70441238-a557-473d-adb6-47024a1b2275 kind: Event lastTimestamp: "2026-04-16T15:27:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:07Z" name: error-404-isvc-1fc59-predictor.18a6dfd5d5abf0c4 namespace: kserve-ci-e2e-test resourceVersion: "25548" uid: ecbddaed-648d-4205-82fe-8ef93dd69aaa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1fc59-predictor namespace: kserve-ci-e2e-test resourceVersion: "25255" uid: 70441238-a557-473d-adb6-47024a1b2275 kind: Event lastTimestamp: "2026-04-16T15:27:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:07Z" name: error-404-isvc-1fc59-predictor.18a6dfd5d6099893 namespace: kserve-ci-e2e-test resourceVersion: "25549" uid: e51be6c4-28eb-425f-8558-af8e793acace reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:27:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1fc59-predictor namespace: kserve-ci-e2e-test resourceVersion: "25435" uid: 70441238-a557-473d-adb6-47024a1b2275 kind: Event lastTimestamp: "2026-04-16T15:27:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:37Z" name: error-404-isvc-1fc59-predictor.18a6dfdcd3751a93 namespace: kserve-ci-e2e-test resourceVersion: "25935" uid: 68756b26-229a-42d1-88e8-65d9a10dc392 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:27:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1fc59-predictor namespace: kserve-ci-e2e-test resourceVersion: "25435" uid: 70441238-a557-473d-adb6-47024a1b2275 kind: Event lastTimestamp: "2026-04-16T15:27:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:37Z" name: error-404-isvc-1fc59-predictor.18a6dfdcd3da7022 namespace: kserve-ci-e2e-test resourceVersion: "25938" uid: c52aae73-41c3-4896-80f8-619ed5094c1d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1fc59 namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: 4629a8f6-d63b-4050-8353-9acc6d88fc57 kind: Event lastTimestamp: "2026-04-16T15:27:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:27:29Z" name: error-404-isvc-1fc59.18a6dfd2570f1141 namespace: kserve-ci-e2e-test resourceVersion: "25854" uid: a1f134d1-fede-450d-93a2-10972c73c8d3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1fc59 namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: 4629a8f6-d63b-4050-8353-9acc6d88fc57 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-1fc59": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1fc59": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:26:37Z" name: error-404-isvc-1fc59.18a6dfd2578e7a4c namespace: kserve-ci-e2e-test resourceVersion: "25264" uid: 7b70cae7-3ae9-45b0-8168-f7bb71169501 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1fc59 namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: 4629a8f6-d63b-4050-8353-9acc6d88fc57 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1fc59": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:26:37Z" name: error-404-isvc-1fc59.18a6dfd2578edb0e namespace: kserve-ci-e2e-test resourceVersion: "25267" uid: 0f0be19d-ca66-4f1f-935f-190db0ac2937 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1fc59 namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: 4629a8f6-d63b-4050-8353-9acc6d88fc57 kind: Event lastTimestamp: "2026-04-16T15:27:29Z" message: InferenceService [error-404-isvc-1fc59] is Ready metadata: creationTimestamp: "2026-04-16T15:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:27:29Z" name: error-404-isvc-1fc59.18a6dfde810ca4e8 namespace: kserve-ci-e2e-test resourceVersion: "25853" uid: a966208e-ee6e-4e44-b860-d4de3ae21ea1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:07:27.343932Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: 30de4458-740a-4852-bfa7-6a75cea3e464 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2d44a-predictor-6bd887658-bj6zv to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:07:27Z" name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv.18a6dec689f92740 namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 3f8f48eb-1575-4022-9012-0d1177827044 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: 30de4458-740a-4852-bfa7-6a75cea3e464 kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:07:27Z" name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv.18a6dec691b9dfd0 namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: ad3c04b9-f330-43f9-8561-eece1875950b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: 30de4458-740a-4852-bfa7-6a75cea3e464 kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:27Z" name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv.18a6dec693319b99 namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: c0ba5af9-a4e3-4644-94bb-1fe67fd30c1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: 30de4458-740a-4852-bfa7-6a75cea3e464 kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:27Z" name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv.18a6dec696c90a22 namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: 89aada90-252f-41ec-83bd-563585bb4c8c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: 30de4458-740a-4852-bfa7-6a75cea3e464 kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:27Z" name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv.18a6dec6973a92ca namespace: kserve-ci-e2e-test resourceVersion: "15918" uid: 4da62432-54f6-4c55-a5d9-d9ed7511893f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: 30de4458-740a-4852-bfa7-6a75cea3e464 kind: Event lastTimestamp: "2026-04-16T15:08:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:08Z" name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv.18a6dec6a0dca986 namespace: kserve-ci-e2e-test resourceVersion: "16331" uid: 0c618f8e-35b0-494c-aa0f-50611f83b017 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: 30de4458-740a-4852-bfa7-6a75cea3e464 kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:52Z" name: error-404-isvc-2d44a-predictor-6bd887658-bj6zv.18a6df4a0ee1750c namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: fbbf57ca-769d-4f40-bb66-92d3c0ad8f4c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2d44a-predictor-6bd887658 namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: ecb0b3aa-5545-4c49-b087-edcec367ef28 kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: 'Created pod: error-404-isvc-2d44a-predictor-6bd887658-bj6zv' metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:27Z" name: error-404-isvc-2d44a-predictor-6bd887658.18a6dec689a21e4a namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: eb0b0736-1f76-419c-9568-2d6cc1871a25 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2d44a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15855" uid: 550225b8-4c9a-4b5c-acfb-802339e00a4b kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: Scaled up replica set error-404-isvc-2d44a-predictor-6bd887658 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:27Z" name: error-404-isvc-2d44a-predictor.18a6dec687f8e74b namespace: kserve-ci-e2e-test resourceVersion: "15864" uid: 023406e7-5790-4a43-82e5-24e22eee5c1a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:07:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2d44a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: 7aeaa8c5-c75f-4f6f-a111-6da21dc7d32f kind: Event lastTimestamp: "2026-04-16T15:08:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:12Z" name: error-404-isvc-2d44a-predictor.18a6deca069b0976 namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: 87df589d-f873-462f-8d96-f4b61a1ff8dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:07:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2d44a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: 7aeaa8c5-c75f-4f6f-a111-6da21dc7d32f kind: Event lastTimestamp: "2026-04-16T15:08:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:12Z" name: error-404-isvc-2d44a-predictor.18a6deca06f624c0 namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: c06ff096-0cb5-40c5-8356-8151fd577b11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:08:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2d44a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 7aeaa8c5-c75f-4f6f-a111-6da21dc7d32f kind: Event lastTimestamp: "2026-04-16T15:08:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:42Z" name: error-404-isvc-2d44a-predictor.18a6ded48308b624 namespace: kserve-ci-e2e-test resourceVersion: "16636" uid: 14d16b5f-6ab3-42f0-8a04-9a9309be11bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:08:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2d44a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 7aeaa8c5-c75f-4f6f-a111-6da21dc7d32f kind: Event lastTimestamp: "2026-04-16T15:08:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:42Z" name: error-404-isvc-2d44a-predictor.18a6ded48363392b namespace: kserve-ci-e2e-test resourceVersion: "16637" uid: 8fbb5569-3188-41cf-b55d-2ceae7f2e29e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2d44a namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: a0c49b5b-4bfc-4a5a-bbbe-83df6277cd68 kind: Event lastTimestamp: "2026-04-16T15:08:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:08:18Z" name: error-404-isvc-2d44a.18a6dec68801d3e4 namespace: kserve-ci-e2e-test resourceVersion: "16426" uid: 4ab14204-0961-48a4-8553-82e4b2a4587b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2d44a namespace: kserve-ci-e2e-test resourceVersion: "16424" uid: a0c49b5b-4bfc-4a5a-bbbe-83df6277cd68 kind: Event lastTimestamp: "2026-04-16T15:08:18Z" message: InferenceService [error-404-isvc-2d44a] is Ready metadata: creationTimestamp: "2026-04-16T15:08:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:08:18Z" name: error-404-isvc-2d44a.18a6ded2843c769a namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: 1eeb1d5a-dc46-4aa5-9a35-f829371c4891 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:16:12.355177Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x namespace: kserve-ci-e2e-test resourceVersion: "19794" uid: b1e28c84-038a-4b26-bbf7-d6551e412327 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-36a90-predictor-574bcd5574-cmn2x to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:16:12Z" name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x.18a6df40c71502d3 namespace: kserve-ci-e2e-test resourceVersion: "19800" uid: ee792104-05b0-4f6d-a843-45e56c9c296e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x namespace: kserve-ci-e2e-test resourceVersion: "19804" uid: b1e28c84-038a-4b26-bbf7-d6551e412327 kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:16:12Z" name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x.18a6df40ce9fb376 namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: 31a0bd60-11a4-4752-b22b-68a0642bff69 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x namespace: kserve-ci-e2e-test resourceVersion: "19795" uid: b1e28c84-038a-4b26-bbf7-d6551e412327 kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:12Z" name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x.18a6df40cfc6bda8 namespace: kserve-ci-e2e-test resourceVersion: "19817" uid: eff71bfa-411d-4b5b-8a73-3d74b382dae8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x namespace: kserve-ci-e2e-test resourceVersion: "19795" uid: b1e28c84-038a-4b26-bbf7-d6551e412327 kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:12Z" name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x.18a6df40d30279ea namespace: kserve-ci-e2e-test resourceVersion: "19820" uid: 0eb7fd0a-a971-4ca8-80cb-76db97712a16 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x namespace: kserve-ci-e2e-test resourceVersion: "19795" uid: b1e28c84-038a-4b26-bbf7-d6551e412327 kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:12Z" name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x.18a6df40d374f0d6 namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: ed93115f-fb7d-4c5f-8947-d56e4c58965a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x namespace: kserve-ci-e2e-test resourceVersion: "19795" uid: b1e28c84-038a-4b26-bbf7-d6551e412327 kind: Event lastTimestamp: "2026-04-16T15:17:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:23Z" name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x.18a6df40d3a92b9b namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: c3349e1e-c844-4503-b4b8-5e1aacbacbd1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x namespace: kserve-ci-e2e-test resourceVersion: "19795" uid: b1e28c84-038a-4b26-bbf7-d6551e412327 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:22Z" name: error-404-isvc-36a90-predictor-574bcd5574-cmn2x.18a6df5122835e8c namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: 031e075a-b575-4c4b-859d-c39c5bca18f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-36a90-predictor-574bcd5574 namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 9c288442-e5d3-480e-96f6-6eb19c60a6cc kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: 'Created pod: error-404-isvc-36a90-predictor-574bcd5574-cmn2x' metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:12Z" name: error-404-isvc-36a90-predictor-574bcd5574.18a6df40c6c2cbf3 namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: b72651c5-853c-4a6d-a45d-b0741465145e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-36a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: a0cf5d6d-68e1-4c19-8756-ec453b087ecb kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: Scaled up replica set error-404-isvc-36a90-predictor-574bcd5574 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:12Z" name: error-404-isvc-36a90-predictor.18a6df40c4fd66e8 namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: ff62e4a6-c588-483a-940d-78a2909862fc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:16:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-36a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: cb61852d-9296-45ad-b368-eb171635277b kind: Event lastTimestamp: "2026-04-16T15:16:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:57Z" name: error-404-isvc-36a90-predictor.18a6df4443cb6ece namespace: kserve-ci-e2e-test resourceVersion: "20306" uid: 20b3daa5-91c6-4a6a-aaca-9d610278644d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:16:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-36a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: cb61852d-9296-45ad-b368-eb171635277b kind: Event lastTimestamp: "2026-04-16T15:16:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:57Z" name: error-404-isvc-36a90-predictor.18a6df444426386b namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: 0331aa65-3536-4e69-bee7-c1edbfd21182 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-36a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: cb61852d-9296-45ad-b368-eb171635277b kind: Event lastTimestamp: "2026-04-16T15:17:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:17:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:12Z" name: error-404-isvc-36a90-predictor.18a6df4ec042f4b1 namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: 1eb5a1d4-99f1-43a9-bc6b-8df984e7c067 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-36a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: cb61852d-9296-45ad-b368-eb171635277b kind: Event lastTimestamp: "2026-04-16T15:17:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:17:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:12Z" name: error-404-isvc-36a90-predictor.18a6df4ec0bbca6f namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: 54906a35-df3d-4c8e-81ae-c08f48751b69 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-36a90 namespace: kserve-ci-e2e-test resourceVersion: "19767" uid: ff599ca9-21be-4408-b9b2-cdd3c6daab08 kind: Event lastTimestamp: "2026-04-16T15:17:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:17:03Z" name: error-404-isvc-36a90.18a6df40c5333942 namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: 5d5a72e1-0e14-49c2-aff9-7656902184d3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-36a90 namespace: kserve-ci-e2e-test resourceVersion: "19767" uid: ff599ca9-21be-4408-b9b2-cdd3c6daab08 kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-36a90": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-36a90": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:16:12Z" name: error-404-isvc-36a90.18a6df40c5b8f766 namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 3dbfd865-64be-4931-a5ef-95410fdb8469 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-36a90 namespace: kserve-ci-e2e-test resourceVersion: "19767" uid: ff599ca9-21be-4408-b9b2-cdd3c6daab08 kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-36a90": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:16:12Z" name: error-404-isvc-36a90.18a6df40c5b93a6f namespace: kserve-ci-e2e-test resourceVersion: "19793" uid: 54220579-5683-4e32-bf84-d985c9e2adb6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-36a90 namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: ff599ca9-21be-4408-b9b2-cdd3c6daab08 kind: Event lastTimestamp: "2026-04-16T15:17:03Z" message: InferenceService [error-404-isvc-36a90] is Ready metadata: creationTimestamp: "2026-04-16T15:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:17:03Z" name: error-404-isvc-36a90.18a6df4cb7a9b6be namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 100fca57-b48c-4697-8ec7-5151f9d56de2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:27:47.866652Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 0d392b18-8ca4-4a19-a5d1-f781e0ef1cea kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:27:47Z" name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r.18a6dfe2b6cbdefa namespace: kserve-ci-e2e-test resourceVersion: "26131" uid: d92934f3-ebf9-4659-a611-c68861bf34ef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r namespace: kserve-ci-e2e-test resourceVersion: "26136" uid: 0d392b18-8ca4-4a19-a5d1-f781e0ef1cea kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: Add eth0 [10.134.0.53/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:27:47Z" name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r.18a6dfe2bdffde2e namespace: kserve-ci-e2e-test resourceVersion: "26146" uid: d0d793b3-3c85-4e30-b3c1-1bd0048aa7b1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 0d392b18-8ca4-4a19-a5d1-f781e0ef1cea kind: Event lastTimestamp: "2026-04-16T15:27:48Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:27:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:48Z" name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r.18a6dfe2bf14b32a namespace: kserve-ci-e2e-test resourceVersion: "26150" uid: 243bc90c-060c-4c03-87c2-632c705118cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 0d392b18-8ca4-4a19-a5d1-f781e0ef1cea kind: Event lastTimestamp: "2026-04-16T15:27:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:27:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:48Z" name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r.18a6dfe2c282e38d namespace: kserve-ci-e2e-test resourceVersion: "26153" uid: a3374fd4-07a0-4d0c-9ad5-8aedb9d3d4b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 0d392b18-8ca4-4a19-a5d1-f781e0ef1cea kind: Event lastTimestamp: "2026-04-16T15:27:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:27:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:48Z" name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r.18a6dfe2c30719a9 namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: a6f96a64-c6b9-499e-9a34-819d78762419 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:27:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 0d392b18-8ca4-4a19-a5d1-f781e0ef1cea kind: Event lastTimestamp: "2026-04-16T15:28:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.53:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:27:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:28:29Z" name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r.18a6dfe2f972b4d5 namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: 958247d0-6e5d-4aae-9739-e88d36bb6f68 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:37:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 0d392b18-8ca4-4a19-a5d1-f781e0ef1cea kind: Event lastTimestamp: "2026-04-16T15:37:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:37:02Z" name: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r.18a6e063e8299b45 namespace: kserve-ci-e2e-test resourceVersion: "30218" uid: 8ce0901e-777d-44e2-bf0f-60f97ee50175 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-41c6b-predictor-687c7b4cb4 namespace: kserve-ci-e2e-test resourceVersion: "26116" uid: edff4a05-ef38-4749-88f6-5f722b973400 kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: 'Created pod: error-404-isvc-41c6b-predictor-687c7b4cb4-w7l5r' metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:47Z" name: error-404-isvc-41c6b-predictor-687c7b4cb4.18a6dfe2b66ad25a namespace: kserve-ci-e2e-test resourceVersion: "26128" uid: d77c7cad-45ee-45af-a0a6-2057598ac7ac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-41c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26113" uid: ab92343e-b57f-45a9-8662-90622d2ea513 kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: Scaled up replica set error-404-isvc-41c6b-predictor-687c7b4cb4 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:47Z" name: error-404-isvc-41c6b-predictor.18a6dfe2b52af5a7 namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 0519e02d-a5de-4310-9b02-47b34a010194 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-41c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26117" uid: 8eab1063-5550-4b41-aafc-241b065dadea kind: Event lastTimestamp: "2026-04-16T15:28:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:02Z" name: error-404-isvc-41c6b-predictor.18a6dfe633c755d3 namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 9e9a4855-aed1-4d24-b017-f8004d29b34a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-41c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26117" uid: 8eab1063-5550-4b41-aafc-241b065dadea kind: Event lastTimestamp: "2026-04-16T15:28:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:02Z" name: error-404-isvc-41c6b-predictor.18a6dfe63407c780 namespace: kserve-ci-e2e-test resourceVersion: "26305" uid: 62dc9d98-dc1e-4c59-b051-f50b0784ab1a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:28:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-41c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26304" uid: 8eab1063-5550-4b41-aafc-241b065dadea kind: Event lastTimestamp: "2026-04-16T15:28:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:28:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:47Z" name: error-404-isvc-41c6b-predictor.18a6dfe9b29a2d39 namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: 3df8b63c-a52d-43de-bd61-a1568ee2401e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:28:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-41c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26304" uid: 8eab1063-5550-4b41-aafc-241b065dadea kind: Event lastTimestamp: "2026-04-16T15:28:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:28:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:47Z" name: error-404-isvc-41c6b-predictor.18a6dfe9b2f3e442 namespace: kserve-ci-e2e-test resourceVersion: "26753" uid: b5d53f58-092d-48c0-bfc2-baeb9763e970 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-41c6b namespace: kserve-ci-e2e-test resourceVersion: "26091" uid: 50fd8d5e-4518-4d69-a670-9ea397e92f56 kind: Event lastTimestamp: "2026-04-16T15:28:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:28:40Z" name: error-404-isvc-41c6b.18a6dfe2b530962d namespace: kserve-ci-e2e-test resourceVersion: "26661" uid: 79b3e41c-17aa-434f-b38e-9cc194917461 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-41c6b namespace: kserve-ci-e2e-test resourceVersion: "26091" uid: 50fd8d5e-4518-4d69-a670-9ea397e92f56 kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: 'Failed to update status for InferenceService "error-404-isvc-41c6b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-41c6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:27:47Z" name: error-404-isvc-41c6b.18a6dfe2b59de146 namespace: kserve-ci-e2e-test resourceVersion: "26124" uid: de54faa0-b82f-4c49-9dec-a1f13d988211 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-41c6b namespace: kserve-ci-e2e-test resourceVersion: "26091" uid: 50fd8d5e-4518-4d69-a670-9ea397e92f56 kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-41c6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:27:47Z" name: error-404-isvc-41c6b.18a6dfe2b59e25ee namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 53976cf1-70d8-4603-b1da-0ca55da80b6a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-41c6b namespace: kserve-ci-e2e-test resourceVersion: "26658" uid: 50fd8d5e-4518-4d69-a670-9ea397e92f56 kind: Event lastTimestamp: "2026-04-16T15:28:40Z" message: InferenceService [error-404-isvc-41c6b] is Ready metadata: creationTimestamp: "2026-04-16T15:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:28:40Z" name: error-404-isvc-41c6b.18a6dfeedcf19550 namespace: kserve-ci-e2e-test resourceVersion: "26659" uid: 58c8bf4b-6fd0-4351-9b2d-4a0fb7506d62 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:27:27.398167Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m namespace: kserve-ci-e2e-test resourceVersion: "25792" uid: 2960b44a-878f-4508-84be-d6eb2332aae4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:27:27Z" name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m.18a6dfddf2c79956 namespace: kserve-ci-e2e-test resourceVersion: "25796" uid: 0b507d92-804d-4060-a04f-fde716e8ffbc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 2960b44a-878f-4508-84be-d6eb2332aae4 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: Add eth0 [10.134.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:27:27Z" name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m.18a6dfddf9d98c5f namespace: kserve-ci-e2e-test resourceVersion: "25809" uid: d6817a90-d9cf-4689-a0cb-13533523880f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m namespace: kserve-ci-e2e-test resourceVersion: "25793" uid: 2960b44a-878f-4508-84be-d6eb2332aae4 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:27Z" name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m.18a6dfddfb01fa2a namespace: kserve-ci-e2e-test resourceVersion: "25811" uid: 801ea6ce-e3f2-4fbf-8c45-bbeb0e7c8470 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m namespace: kserve-ci-e2e-test resourceVersion: "25793" uid: 2960b44a-878f-4508-84be-d6eb2332aae4 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:27Z" name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m.18a6dfde02467d74 namespace: kserve-ci-e2e-test resourceVersion: "25817" uid: 704acd88-a312-40db-a74f-cccf507ad03e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m namespace: kserve-ci-e2e-test resourceVersion: "25793" uid: 2960b44a-878f-4508-84be-d6eb2332aae4 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:27Z" name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m.18a6dfde02ba6e3a namespace: kserve-ci-e2e-test resourceVersion: "25818" uid: 914b0ef0-ac1e-4347-b37a-fc97e492004a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m namespace: kserve-ci-e2e-test resourceVersion: "25793" uid: 2960b44a-878f-4508-84be-d6eb2332aae4 kind: Event lastTimestamp: "2026-04-16T15:44:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:44:58Z" name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m.18a6dfde110f0f94 namespace: kserve-ci-e2e-test resourceVersion: "33544" uid: fd8c8fa8-8776-4d6c-8cd5-8116a33d8254 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:44:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m namespace: kserve-ci-e2e-test resourceVersion: "25793" uid: 2960b44a-878f-4508-84be-d6eb2332aae4 kind: Event lastTimestamp: "2026-04-16T15:44:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:44:56Z" name: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m.18a6e0d251780ca9 namespace: kserve-ci-e2e-test resourceVersion: "33495" uid: ec99edca-77ea-4b9d-b3dc-cf56f24a915c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9c7d6-predictor-5fb75bc8f4 namespace: kserve-ci-e2e-test resourceVersion: "25785" uid: f407ec64-a1cf-4485-b967-56caea3dd500 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: 'Created pod: error-404-isvc-9c7d6-predictor-5fb75bc8f4-f4r9m' metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:27Z" name: error-404-isvc-9c7d6-predictor-5fb75bc8f4.18a6dfddf27873f2 namespace: kserve-ci-e2e-test resourceVersion: "25794" uid: 3a8c8cce-f8d7-419d-8d43-ba0d836badaf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9c7d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 9d5ffb1e-a37a-4ab3-a666-81b573e055b3 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: Scaled up replica set error-404-isvc-9c7d6-predictor-5fb75bc8f4 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:27Z" name: error-404-isvc-9c7d6-predictor.18a6dfddf125cd45 namespace: kserve-ci-e2e-test resourceVersion: "25786" uid: c17a1945-b73d-4cde-ab40-8edf4a299027 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c7d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: a5274d75-1bbe-4d11-b71f-613b94eb446c kind: Event lastTimestamp: "2026-04-16T15:27:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:42Z" name: error-404-isvc-9c7d6-predictor.18a6dfe16fd7242d namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: 799cec0b-1b30-4e3f-ab12-3b3cef2ed2e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c7d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: a5274d75-1bbe-4d11-b71f-613b94eb446c kind: Event lastTimestamp: "2026-04-16T15:27:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:42Z" name: error-404-isvc-9c7d6-predictor.18a6dfe170373333 namespace: kserve-ci-e2e-test resourceVersion: "25998" uid: 68eadaba-f848-4876-8b6b-4170f53a6fe7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:27:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c7d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: a5274d75-1bbe-4d11-b71f-613b94eb446c kind: Event lastTimestamp: "2026-04-16T15:28:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:27:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:42Z" name: error-404-isvc-9c7d6-predictor.18a6dfe4eee2b5a4 namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 4a41c627-aabc-45bd-ba22-1c91820703c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:27:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c7d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: a5274d75-1bbe-4d11-b71f-613b94eb446c kind: Event lastTimestamp: "2026-04-16T15:28:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:27:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:42Z" name: error-404-isvc-9c7d6-predictor.18a6dfe4ef3fb37d namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: 0d28ca52-e548-48b8-bf03-78211da2c679 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "25759" uid: 0190a46e-b893-4754-b0b1-cc73fd764d0c kind: Event lastTimestamp: "2026-04-16T15:28:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:28:18Z" name: error-404-isvc-9c7d6.18a6dfddf1311940 namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: 382a1037-1038-48f4-93ba-741892b45fe8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "25759" uid: 0190a46e-b893-4754-b0b1-cc73fd764d0c kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: 'Failed to update status for InferenceService "error-404-isvc-9c7d6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9c7d6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:27:27Z" name: error-404-isvc-9c7d6.18a6dfddf1a03c39 namespace: kserve-ci-e2e-test resourceVersion: "25789" uid: 82f71ceb-bf4a-4c1f-8b5f-54f866acda80 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "25759" uid: 0190a46e-b893-4754-b0b1-cc73fd764d0c kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9c7d6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:27:27Z" name: error-404-isvc-9c7d6.18a6dfddf1a07fac namespace: kserve-ci-e2e-test resourceVersion: "25791" uid: 6df58d8c-7b38-468c-83dd-65ef044021f9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "26449" uid: 0190a46e-b893-4754-b0b1-cc73fd764d0c kind: Event lastTimestamp: "2026-04-16T15:28:18Z" message: InferenceService [error-404-isvc-9c7d6] is Ready metadata: creationTimestamp: "2026-04-16T15:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:28:18Z" name: error-404-isvc-9c7d6.18a6dfe9f4d48558 namespace: kserve-ci-e2e-test resourceVersion: "26450" uid: e1fb6219-c939-430b-92d3-8c18e4cb9c5d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:17:22.634738Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: 71002a64-03d0-4265-99f6-b078b62619c5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b2be2-predictor-5f8c869854-s6kpj to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:17:22Z" name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj.18a6df51241206ee namespace: kserve-ci-e2e-test resourceVersion: "20656" uid: 0522c008-c5c0-408e-85cf-5c87cd09f887 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: 71002a64-03d0-4265-99f6-b078b62619c5 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:17:22Z" name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj.18a6df512b9706e9 namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 5a26b822-d640-4af0-b79b-2f4cb3bd1e52 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 71002a64-03d0-4265-99f6-b078b62619c5 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:22Z" name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj.18a6df512cd4cf7c namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 8b5896ea-91a5-4d95-aae2-cadc93c2b4c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 71002a64-03d0-4265-99f6-b078b62619c5 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:22Z" name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj.18a6df5131e407ed namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: 371290f2-40b9-4adc-8200-59027f1995bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 71002a64-03d0-4265-99f6-b078b62619c5 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:22Z" name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj.18a6df513255d819 namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: f6a0bcdb-8675-44d5-bee1-eb370ff19ee7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:17:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 71002a64-03d0-4265-99f6-b078b62619c5 kind: Event lastTimestamp: "2026-04-16T15:18:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:17:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:04Z" name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj.18a6df516b1c831d namespace: kserve-ci-e2e-test resourceVersion: "21092" uid: f075b5dc-71f4-45e4-914f-1c038632be67 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 71002a64-03d0-4265-99f6-b078b62619c5 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:26:37Z" name: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj.18a6dfd25683cbb5 namespace: kserve-ci-e2e-test resourceVersion: "25253" uid: 456637d7-d81b-46c5-9d6c-a40f4d672965 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b2be2-predictor-5f8c869854 namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: ff00f013-a1bb-40ab-bb35-f05491d75e3d kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: 'Created pod: error-404-isvc-b2be2-predictor-5f8c869854-s6kpj' metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:22Z" name: error-404-isvc-b2be2-predictor-5f8c869854.18a6df5123b3e304 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 6b3bb778-ad63-4ef9-a8e2-bd94a69878ab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b2be2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20639" uid: 31db8e9d-5734-4c16-91cd-0d4f21ad4a5f kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: Scaled up replica set error-404-isvc-b2be2-predictor-5f8c869854 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:22Z" name: error-404-isvc-b2be2-predictor.18a6df51225c5e36 namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 6d576f65-650e-4ab0-8923-550ba0cb17f2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:17:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2be2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 80672bc2-2eab-471f-a754-112a02edebee kind: Event lastTimestamp: "2026-04-16T15:18:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:07Z" name: error-404-isvc-b2be2-predictor.18a6df54a11b7e68 namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: 4831a8cc-823a-45f2-83b6-f522e5102daf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:17:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2be2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 80672bc2-2eab-471f-a754-112a02edebee kind: Event lastTimestamp: "2026-04-16T15:18:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:07Z" name: error-404-isvc-b2be2-predictor.18a6df54a1766bbb namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: 3870b815-ddb2-4fad-b58e-9f60e66f0196 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:18:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2be2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20817" uid: 80672bc2-2eab-471f-a754-112a02edebee kind: Event lastTimestamp: "2026-04-16T15:18:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:18:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:37Z" name: error-404-isvc-b2be2-predictor.18a6df5f1da4e6f5 namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: 5bbbf68c-3dfd-487f-b40e-bf86eae7954b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:18:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2be2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20817" uid: 80672bc2-2eab-471f-a754-112a02edebee kind: Event lastTimestamp: "2026-04-16T15:18:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:18:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:37Z" name: error-404-isvc-b2be2-predictor.18a6df5f1e03580c namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 65ffc635-134a-4a29-b86e-0f6c226c8dd4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: bb216e2e-4923-4b3a-8c30-f7e5bf781de1 kind: Event lastTimestamp: "2026-04-16T15:18:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:14Z" name: error-404-isvc-b2be2.18a6df5122743fa6 namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: 30cadf6b-60c0-48d4-bd20-c17cb3fc43ee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: bb216e2e-4923-4b3a-8c30-f7e5bf781de1 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: 'Failed to update status for InferenceService "error-404-isvc-b2be2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b2be2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:17:22Z" name: error-404-isvc-b2be2.18a6df5122f6774f namespace: kserve-ci-e2e-test resourceVersion: "20649" uid: dd9af9cd-ab07-49e7-9f12-fb717b6e830e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: bb216e2e-4923-4b3a-8c30-f7e5bf781de1 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b2be2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:17:22Z" name: error-404-isvc-b2be2.18a6df5122f6c252 namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: 062dbf48-c1a8-4b93-8138-c420aa48388d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "21305" uid: bb216e2e-4923-4b3a-8c30-f7e5bf781de1 kind: Event lastTimestamp: "2026-04-16T15:18:14Z" message: InferenceService [error-404-isvc-b2be2] is Ready metadata: creationTimestamp: "2026-04-16T15:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:14Z" name: error-404-isvc-b2be2.18a6df5d4ec13069 namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: 86853957-4c52-4728-86d4-131a0d201b3f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:16:52.231872Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5 namespace: kserve-ci-e2e-test resourceVersion: "20215" uid: a7e59109-5394-4ef9-bf06-0bfc70ce9b63 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5 to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:16:52Z" name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5.18a6df4a0feb7a7b namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 28c1dca1-9c96-42b7-812a-aeeb6dc35b8f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5 namespace: kserve-ci-e2e-test resourceVersion: "20225" uid: a7e59109-5394-4ef9-bf06-0bfc70ce9b63 kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:16:52Z" name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5.18a6df4a1764febf namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: fc8de0e8-6ab0-4b2e-bf2d-d38e5dd706f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5 namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: a7e59109-5394-4ef9-bf06-0bfc70ce9b63 kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:52Z" name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5.18a6df4a1886c186 namespace: kserve-ci-e2e-test resourceVersion: "20238" uid: 844fede6-a9ed-4d01-a7b9-901278d84a38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5 namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: a7e59109-5394-4ef9-bf06-0bfc70ce9b63 kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:52Z" name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5.18a6df4a1d853fc6 namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: d364eaeb-d462-4b86-adac-61c710269f22 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5 namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: a7e59109-5394-4ef9-bf06-0bfc70ce9b63 kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:52Z" name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5.18a6df4a1e0062db namespace: kserve-ci-e2e-test resourceVersion: "20244" uid: 43bb7854-a5a3-42d8-96b5-cc3a6a23031f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5 namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: a7e59109-5394-4ef9-bf06-0bfc70ce9b63 kind: Event lastTimestamp: "2026-04-16T15:18:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:13Z" name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5.18a6df4a2cb06f46 namespace: kserve-ci-e2e-test resourceVersion: "21284" uid: 4dfff2df-990a-413c-bb23-3086006a4b92 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5 namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: a7e59109-5394-4ef9-bf06-0bfc70ce9b63 kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:12Z" name: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5.18a6df5cc05e9ce4 namespace: kserve-ci-e2e-test resourceVersion: "21234" uid: 3b1d062c-852b-4bc4-969d-20ab560b27fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b8642-predictor-54bb7c4df6 namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: 4e546b91-4004-4ce9-93ae-dda4a971a2a9 kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: 'Created pod: error-404-isvc-b8642-predictor-54bb7c4df6-kbbt5' metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:52Z" name: error-404-isvc-b8642-predictor-54bb7c4df6.18a6df4a0f4e096e namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: 90b24c5a-83df-4b91-ba48-2d5839b08c06 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b8642-predictor namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 91be1c4a-0022-4f01-b81f-a7fa5cf4dad3 kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: Scaled up replica set error-404-isvc-b8642-predictor-54bb7c4df6 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:52Z" name: error-404-isvc-b8642-predictor.18a6df4a0e07d979 namespace: kserve-ci-e2e-test resourceVersion: "20207" uid: d239a2ab-b8b0-4083-b276-60dfa488b0a9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:17:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b8642-predictor namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 2e8fff23-c5df-42ac-8cdf-1fed7efd45b6 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:22Z" name: error-404-isvc-b8642-predictor.18a6df4d8d33be7e namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: 9463385d-c99e-44bd-ba14-2d53c90d51a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:17:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b8642-predictor namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 2e8fff23-c5df-42ac-8cdf-1fed7efd45b6 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:22Z" name: error-404-isvc-b8642-predictor.18a6df4d8d8ef4c9 namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 9c684059-4087-4acd-ba6b-6d2374ff61c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:17:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b8642-predictor namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: 2e8fff23-c5df-42ac-8cdf-1fed7efd45b6 kind: Event lastTimestamp: "2026-04-16T15:18:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:07Z" name: error-404-isvc-b8642-predictor.18a6df548ae61c7a namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: f0a2611f-3fb5-427f-a6aa-30fb0856a25b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:17:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b8642-predictor namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: 2e8fff23-c5df-42ac-8cdf-1fed7efd45b6 kind: Event lastTimestamp: "2026-04-16T15:18:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:07Z" name: error-404-isvc-b8642-predictor.18a6df548b44efb7 namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: 6cc16c2e-5042-4fab-a9c5-5418c66e2f83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b8642 namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: e3700c0f-37eb-4702-a565-ca1ed0447812 kind: Event lastTimestamp: "2026-04-16T15:17:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:17:43Z" name: error-404-isvc-b8642.18a6df4a0e3d8a78 namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: e9af36b2-1e10-4180-b517-3407f01f9b4c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b8642 namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: e3700c0f-37eb-4702-a565-ca1ed0447812 kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: 'Failed to update status for InferenceService "error-404-isvc-b8642": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b8642": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:16:52Z" name: error-404-isvc-b8642.18a6df4a0e8e4683 namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: 1ae30681-dc5f-4a9b-9678-a765f65452a5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b8642 namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: e3700c0f-37eb-4702-a565-ca1ed0447812 kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b8642": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:16:52Z" name: error-404-isvc-b8642.18a6df4a0e8e927c namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: 512008cc-0fe6-4491-83cc-80c28fb53042 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b8642 namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: e3700c0f-37eb-4702-a565-ca1ed0447812 kind: Event lastTimestamp: "2026-04-16T15:17:43Z" message: InferenceService [error-404-isvc-b8642] is Ready metadata: creationTimestamp: "2026-04-16T15:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:17:43Z" name: error-404-isvc-b8642.18a6df5610503fbb namespace: kserve-ci-e2e-test resourceVersion: "20869" uid: cbca40ec-2a13-4d52-a367-7c8b392944a3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:06:47.430090Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7 namespace: kserve-ci-e2e-test resourceVersion: "15361" uid: 0ad5a8c3-7ac8-4abc-9aaf-93066d37d18a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7 to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:06:47Z" name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7.18a6debd3eec4a18 namespace: kserve-ci-e2e-test resourceVersion: "15367" uid: d620610b-1c2d-4f02-81f6-0b9436d8951c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7 namespace: kserve-ci-e2e-test resourceVersion: "15372" uid: 0ad5a8c3-7ac8-4abc-9aaf-93066d37d18a kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:06:47Z" name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7.18a6debd4648712b namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: 8689d212-2b03-4ec1-a747-a4c0dd2d1003 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 0ad5a8c3-7ac8-4abc-9aaf-93066d37d18a kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:47Z" name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7.18a6debd477a6f50 namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: 10c82a74-df91-4f4c-8ab5-30c8c42970e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 0ad5a8c3-7ac8-4abc-9aaf-93066d37d18a kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:47Z" name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7.18a6debd4c93a721 namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: f063a1ea-ff75-411f-bca0-256ecc5e3910 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 0ad5a8c3-7ac8-4abc-9aaf-93066d37d18a kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:47Z" name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7.18a6debd4d0d0875 namespace: kserve-ci-e2e-test resourceVersion: "15393" uid: 48a10df3-91e5-443d-a566-3fdfa28d12a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:06:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 0ad5a8c3-7ac8-4abc-9aaf-93066d37d18a kind: Event lastTimestamp: "2026-04-16T15:07:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:29Z" name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7.18a6debd83efea40 namespace: kserve-ci-e2e-test resourceVersion: "15950" uid: 2c3bd508-ebed-442e-8668-7b83b0b05833 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 0ad5a8c3-7ac8-4abc-9aaf-93066d37d18a kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:12Z" name: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7.18a6df40c29cb0ee namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: f6f0f325-324c-4d6b-984f-27f6fc845c10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c9f8b-predictor-667b95c6b8 namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: eabea6c7-673c-46e8-97a3-b8447d13385e kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: 'Created pod: error-404-isvc-c9f8b-predictor-667b95c6b8-wknl7' metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:47Z" name: error-404-isvc-c9f8b-predictor-667b95c6b8.18a6debd3e7c3a3c namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: fd9c5dd7-2bf1-4cb0-82e6-52e780d6098c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c9f8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: 1bef14ea-ae28-4600-a9a5-719aa7ea59d8 kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: Scaled up replica set error-404-isvc-c9f8b-predictor-667b95c6b8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:47Z" name: error-404-isvc-c9f8b-predictor.18a6debd3d1d28b6 namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: f9e30e2b-45e8-4211-ade2-d10628708eb3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:07:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c9f8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: d3ff103b-d164-47cf-a709-337bd3c75ce7 kind: Event lastTimestamp: "2026-04-16T15:08:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:02Z" name: error-404-isvc-c9f8b-predictor.18a6dec0bbbf0c03 namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: d77ef7cb-37eb-4423-ac72-d49d98a2741e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:07:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c9f8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: d3ff103b-d164-47cf-a709-337bd3c75ce7 kind: Event lastTimestamp: "2026-04-16T15:08:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:02Z" name: error-404-isvc-c9f8b-predictor.18a6dec0bc18d9e6 namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: e14ce50d-1c2a-48a3-8e7e-a9e36f00cabf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:07:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c9f8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: d3ff103b-d164-47cf-a709-337bd3c75ce7 kind: Event lastTimestamp: "2026-04-16T15:07:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:47Z" name: error-404-isvc-c9f8b-predictor.18a6dec43aa54d09 namespace: kserve-ci-e2e-test resourceVersion: "16114" uid: 0021dc05-940d-4986-b145-b0909a067c91 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:07:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c9f8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: d3ff103b-d164-47cf-a709-337bd3c75ce7 kind: Event lastTimestamp: "2026-04-16T15:07:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:47Z" name: error-404-isvc-c9f8b-predictor.18a6dec43afa6397 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 606a98cc-9ddb-4d08-a027-e7dd6eb559a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c9f8b namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: 1e524cd8-6481-4f9d-bbee-334f070dcb5f kind: Event lastTimestamp: "2026-04-16T15:07:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:07:39Z" name: error-404-isvc-c9f8b.18a6debd3d377d21 namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: 73540330-0d86-40f6-a50d-b45c2ba8ab12 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c9f8b namespace: kserve-ci-e2e-test resourceVersion: "16050" uid: 1e524cd8-6481-4f9d-bbee-334f070dcb5f kind: Event lastTimestamp: "2026-04-16T15:07:39Z" message: InferenceService [error-404-isvc-c9f8b] is Ready metadata: creationTimestamp: "2026-04-16T15:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:07:39Z" name: error-404-isvc-c9f8b.18a6dec9678fea6d namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: e929f1cb-6aa8-4c54-9d87-aa6e63de9dbf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:18:12.533464Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8 namespace: kserve-ci-e2e-test resourceVersion: "21241" uid: edebcbcd-363d-4468-880a-45829d6ec699 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-cee76-predictor-866cc8b64-4mdx8 to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:18:12Z" name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8.18a6df5cc2441f00 namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 5666a637-735e-4668-8ade-e4f697df7ca3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8 namespace: kserve-ci-e2e-test resourceVersion: "21249" uid: edebcbcd-363d-4468-880a-45829d6ec699 kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:18:12Z" name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8.18a6df5cc9eeb69f namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 6a60d853-4a8b-4502-a536-d35ad891bbff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: edebcbcd-363d-4468-880a-45829d6ec699 kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:12Z" name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8.18a6df5ccb0dc87f namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: f6c35137-78a5-4528-b7f9-a690e82e0d29 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: edebcbcd-363d-4468-880a-45829d6ec699 kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:12Z" name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8.18a6df5ccfc0986d namespace: kserve-ci-e2e-test resourceVersion: "21268" uid: 91f57b7a-6060-4c07-a75f-78ef8775f71c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: edebcbcd-363d-4468-880a-45829d6ec699 kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:12Z" name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8.18a6df5cd03430b0 namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: 43393702-ff7e-4846-bd16-e11bff597db3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: edebcbcd-363d-4468-880a-45829d6ec699 kind: Event lastTimestamp: "2026-04-16T15:18:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:54Z" name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8.18a6df5cde027908 namespace: kserve-ci-e2e-test resourceVersion: "21663" uid: 43c98863-f2f9-4920-b52e-0b1bed712eae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: edebcbcd-363d-4468-880a-45829d6ec699 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:27Z" name: error-404-isvc-cee76-predictor-866cc8b64-4mdx8.18a6dfddf0a7dba9 namespace: kserve-ci-e2e-test resourceVersion: "25783" uid: 039410b6-0d00-4884-898a-c17458e731e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-cee76-predictor-866cc8b64 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: c685daad-6e34-4303-ac93-3503dfa5a15c kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: 'Created pod: error-404-isvc-cee76-predictor-866cc8b64-4mdx8' metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:12Z" name: error-404-isvc-cee76-predictor-866cc8b64.18a6df5cc1c642ba namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: 3e16963c-1c6f-4044-9f05-bb28b4368e46 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-cee76-predictor namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 9d3ea37b-6428-4ce0-bf16-a4ed29a7cc28 kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: Scaled up replica set error-404-isvc-cee76-predictor-866cc8b64 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:12Z" name: error-404-isvc-cee76-predictor.18a6df5cc062bf0d namespace: kserve-ci-e2e-test resourceVersion: "21235" uid: 3b6cc4c7-7df8-47a2-8595-e590f0f02838 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cee76-predictor namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: 92d46466-0645-42ec-9dd1-05770593d966 kind: Event lastTimestamp: "2026-04-16T15:18:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:27Z" name: error-404-isvc-cee76-predictor.18a6df603f1ad9bb namespace: kserve-ci-e2e-test resourceVersion: "21451" uid: 9384e637-283c-4dfa-bc40-0d820c44a644 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cee76-predictor namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: 92d46466-0645-42ec-9dd1-05770593d966 kind: Event lastTimestamp: "2026-04-16T15:18:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:27Z" name: error-404-isvc-cee76-predictor.18a6df603f7ab536 namespace: kserve-ci-e2e-test resourceVersion: "21453" uid: 330172fd-88e8-4e48-970c-9556da19b061 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:18:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cee76-predictor namespace: kserve-ci-e2e-test resourceVersion: "21452" uid: 92d46466-0645-42ec-9dd1-05770593d966 kind: Event lastTimestamp: "2026-04-16T15:19:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:18:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:19:27Z" name: error-404-isvc-cee76-predictor.18a6df63be259cc4 namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: 0a5af9dd-d951-4839-8506-bfc623d32fb7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:18:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cee76-predictor namespace: kserve-ci-e2e-test resourceVersion: "21452" uid: 92d46466-0645-42ec-9dd1-05770593d966 kind: Event lastTimestamp: "2026-04-16T15:19:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:18:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:19:27Z" name: error-404-isvc-cee76-predictor.18a6df63be82eccc namespace: kserve-ci-e2e-test resourceVersion: "21973" uid: 06139c74-f42b-4df7-84ae-698c2ff557b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: 7bbb1f41-6a9a-4b07-a758-029b27094804 kind: Event lastTimestamp: "2026-04-16T15:19:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:19:04Z" name: error-404-isvc-cee76.18a6df5cc06bdaed namespace: kserve-ci-e2e-test resourceVersion: "21753" uid: fd6bae5e-8859-43af-b9a5-d6568f415b28 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: 7bbb1f41-6a9a-4b07-a758-029b27094804 kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-cee76": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cee76": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:12Z" name: error-404-isvc-cee76.18a6df5cc0efd78e namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: 4576d508-b46d-4a94-a6ea-6cb30f677546 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: 7bbb1f41-6a9a-4b07-a758-029b27094804 kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cee76": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:12Z" name: error-404-isvc-cee76.18a6df5cc0f00f4b namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: e9029e91-bfd8-4b4b-aa61-09f26747e010 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:19:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21751" uid: 7bbb1f41-6a9a-4b07-a758-029b27094804 kind: Event lastTimestamp: "2026-04-16T15:19:04Z" message: InferenceService [error-404-isvc-cee76] is Ready metadata: creationTimestamp: "2026-04-16T15:19:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:19:04Z" name: error-404-isvc-cee76.18a6df68c1718e69 namespace: kserve-ci-e2e-test resourceVersion: "21752" uid: 348260b8-2440-48b2-9be1-84ec25b3ada0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:05:17.249789Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn namespace: kserve-ci-e2e-test resourceVersion: "14344" uid: 70861a45-bf20-471b-abae-c5a854e55276 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d1f99-predictor-6f8c58666-kmthn to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:05:17Z" name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn.18a6dea83fc213db namespace: kserve-ci-e2e-test resourceVersion: "14349" uid: ae2c6267-504d-4048-8659-92c84a8d8546 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 70861a45-bf20-471b-abae-c5a854e55276 kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:05:17Z" name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn.18a6dea847a954be namespace: kserve-ci-e2e-test resourceVersion: "14391" uid: 6657b2ac-d7eb-4e52-8cdf-0c47f428c745 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: 70861a45-bf20-471b-abae-c5a854e55276 kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1407" metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:17Z" name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn.18a6dea848bf74fb namespace: kserve-ci-e2e-test resourceVersion: "14395" uid: a2dde70a-01db-463a-84aa-48af6de537c5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: 70861a45-bf20-471b-abae-c5a854e55276 kind: Event lastTimestamp: "2026-04-16T15:05:30Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1407" in 13.443s (13.443s including waiting). Image size: 1333455482 bytes.' metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:30Z" name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn.18a6deab6a09da16 namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 1534de72-2c2c-46d9-91ca-61d312fc6867 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: 70861a45-bf20-471b-abae-c5a854e55276 kind: Event lastTimestamp: "2026-04-16T15:05:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:30Z" name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn.18a6deab7101ad95 namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: 852a1670-1978-4ee6-b916-839f61aa2cc4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: 70861a45-bf20-471b-abae-c5a854e55276 kind: Event lastTimestamp: "2026-04-16T15:05:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:30Z" name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn.18a6deab717af210 namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 6d73eb02-7505-40bf-b61f-115241bc8062 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:05:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: 70861a45-bf20-471b-abae-c5a854e55276 kind: Event lastTimestamp: "2026-04-16T15:06:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:05:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:12Z" name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn.18a6deab85df8b76 namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: 1939b79c-8792-4d0a-a2e1-e7c8193ab59b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: 70861a45-bf20-471b-abae-c5a854e55276 kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:47Z" name: error-404-isvc-d1f99-predictor-6f8c58666-kmthn.18a6debd355d5f82 namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 630d692f-d98e-42bf-aa20-acfc53a4fd44 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d1f99-predictor-6f8c58666 namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: 05dcf6bc-a6eb-4e40-8b2d-e89f7f60d9bf kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: 'Created pod: error-404-isvc-d1f99-predictor-6f8c58666-kmthn' metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:17Z" name: error-404-isvc-d1f99-predictor-6f8c58666.18a6dea83f153fd7 namespace: kserve-ci-e2e-test resourceVersion: "14345" uid: 1aa6c7ee-7b00-4eb8-a3c1-d5b875edd477 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d1f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: 1ecc7600-96f4-48a2-82e8-fe35752ca12f kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: Scaled up replica set error-404-isvc-d1f99-predictor-6f8c58666 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:17Z" name: error-404-isvc-d1f99-predictor.18a6dea83cfddc6f namespace: kserve-ci-e2e-test resourceVersion: "14339" uid: 9235d0aa-faae-4fce-a1fb-caefcc198060 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:05:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: d8a62ade-ee1b-42ad-845c-02286bc28724 kind: Event lastTimestamp: "2026-04-16T15:06:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:02Z" name: error-404-isvc-d1f99-predictor.18a6deabbb8af739 namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 560eb92c-2258-41e3-b9db-e3a028d49c71 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:05:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: d8a62ade-ee1b-42ad-845c-02286bc28724 kind: Event lastTimestamp: "2026-04-16T15:06:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:02Z" name: error-404-isvc-d1f99-predictor.18a6deabbbe90498 namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: 5a3ea95e-e150-42c7-882c-554efcfd5681 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: d8a62ade-ee1b-42ad-845c-02286bc28724 kind: Event lastTimestamp: "2026-04-16T15:06:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:32Z" name: error-404-isvc-d1f99-predictor.18a6deb637dfa2e4 namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 53d08545-17ff-419a-a1c7-1214612becba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: d8a62ade-ee1b-42ad-845c-02286bc28724 kind: Event lastTimestamp: "2026-04-16T15:06:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:32Z" name: error-404-isvc-d1f99-predictor.18a6deb6383ba3fd namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 91ee672f-ddde-4063-a2a1-3339ef12f22d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1f99 namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: 39ed8583-e58a-40ca-b684-58907b121323 kind: Event lastTimestamp: "2026-04-16T15:06:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:22Z" name: error-404-isvc-d1f99.18a6dea83cdbfc68 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 3d8888ec-05f0-4655-a8f7-9757dcf4e272 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1f99 namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: 39ed8583-e58a-40ca-b684-58907b121323 kind: Event lastTimestamp: "2026-04-16T15:06:22Z" message: InferenceService [error-404-isvc-d1f99] is Ready metadata: creationTimestamp: "2026-04-16T15:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:22Z" name: error-404-isvc-d1f99.18a6deb769593582 namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: 6965bf89-e5d0-435f-ab25-a4c6c55c4f7d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:05:17.162168Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-695857997-skcx9 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: dbf610c8-a8fa-418e-ba35-19428a90c024 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-695857997-skcx9 to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:05:17Z" name: isvc-sklearn-graph-1-predictor-695857997-skcx9.18a6dea83a891bec namespace: kserve-ci-e2e-test resourceVersion: "14319" uid: c4d94f2a-e6f6-4922-bd8e-e065eb2cc55d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-695857997-skcx9 namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: dbf610c8-a8fa-418e-ba35-19428a90c024 kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:05:17Z" name: isvc-sklearn-graph-1-predictor-695857997-skcx9.18a6dea8538c0f4a namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: 0969df1b-0c2b-4664-b17c-c77230bc57cf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-695857997-skcx9 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: dbf610c8-a8fa-418e-ba35-19428a90c024 kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:17Z" name: isvc-sklearn-graph-1-predictor-695857997-skcx9.18a6dea8559f291a namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: 4de02284-ce04-46e0-aa62-46d7b147451b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-695857997-skcx9 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: dbf610c8-a8fa-418e-ba35-19428a90c024 kind: Event lastTimestamp: "2026-04-16T15:05:30Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" in 13.134s (13.134s including waiting). Image size: 300960724 bytes.' metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:30Z" name: isvc-sklearn-graph-1-predictor-695857997-skcx9.18a6deab647bd6dc namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: 07969077-90e9-4ec2-a74f-21301b182341 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-695857997-skcx9 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: dbf610c8-a8fa-418e-ba35-19428a90c024 kind: Event lastTimestamp: "2026-04-16T15:05:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:30Z" name: isvc-sklearn-graph-1-predictor-695857997-skcx9.18a6deab6797a399 namespace: kserve-ci-e2e-test resourceVersion: "14556" uid: 8ad11833-2170-4ab0-998f-c31b4aa18052 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-695857997-skcx9 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: dbf610c8-a8fa-418e-ba35-19428a90c024 kind: Event lastTimestamp: "2026-04-16T15:05:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:30Z" name: isvc-sklearn-graph-1-predictor-695857997-skcx9.18a6deab680a8a76 namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: c990b6d1-5e53-471f-9a79-0d7d42b992a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-695857997-skcx9 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: dbf610c8-a8fa-418e-ba35-19428a90c024 kind: Event lastTimestamp: "2026-04-16T15:05:35Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1407" metadata: creationTimestamp: "2026-04-16T15:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:35Z" name: isvc-sklearn-graph-1-predictor-695857997-skcx9.18a6deac752dd8ac namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 5472ab6d-c0fd-49b7-ad81-0ba23f585b63 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-695857997-skcx9 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: dbf610c8-a8fa-418e-ba35-19428a90c024 kind: Event lastTimestamp: "2026-04-16T15:05:41Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1407" in 6.325s (6.325s including waiting). Image size: 1560613827 bytes.' metadata: creationTimestamp: "2026-04-16T15:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:41Z" name: isvc-sklearn-graph-1-predictor-695857997-skcx9.18a6deadee3ae027 namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: 096e5c3c-c52c-4c17-806b-0a40a4bae573 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-695857997-skcx9 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: dbf610c8-a8fa-418e-ba35-19428a90c024 kind: Event lastTimestamp: "2026-04-16T15:05:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:41Z" name: isvc-sklearn-graph-1-predictor-695857997-skcx9.18a6deadf180d3e0 namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: d9266ffd-f578-4caf-bf04-c3c68a37e550 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-695857997-skcx9 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: dbf610c8-a8fa-418e-ba35-19428a90c024 kind: Event lastTimestamp: "2026-04-16T15:05:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:41Z" name: isvc-sklearn-graph-1-predictor-695857997-skcx9.18a6deadf1f163f2 namespace: kserve-ci-e2e-test resourceVersion: "14677" uid: 7b6257bc-04ae-4a11-9ead-3ad7b14e7857 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T15:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-695857997-skcx9 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: dbf610c8-a8fa-418e-ba35-19428a90c024 kind: Event lastTimestamp: "2026-04-16T15:06:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:43Z" name: isvc-sklearn-graph-1-predictor-695857997-skcx9.18a6deae181eeafb namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 95beb5b0-4faa-4ba9-bd8b-08d9443fdb96 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-695857997-skcx9 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: dbf610c8-a8fa-418e-ba35-19428a90c024 kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:27Z" name: isvc-sklearn-graph-1-predictor-695857997-skcx9.18a6dec6840bbd1e namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: a3175f7c-4cb7-4679-8f11-d99cf5efe98d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-695857997 namespace: kserve-ci-e2e-test resourceVersion: "14304" uid: 10398a3b-a092-4797-a005-e4c23d7205b2 kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-695857997-skcx9' metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:17Z" name: isvc-sklearn-graph-1-predictor-695857997.18a6dea83a2953e4 namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 1912e2ee-a364-449a-b78f-a6f664610fcb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14303" uid: cc10f6fd-4944-41c1-9faa-bcfa5ec14b6d kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-695857997 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:17Z" name: isvc-sklearn-graph-1-predictor.18a6dea8379e0f1c namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 0bdda49f-3724-4984-996d-3670f3afaa44 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:05:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: 4f4bcbb0-65e1-4a08-a6d2-0fa549291381 kind: Event lastTimestamp: "2026-04-16T15:05:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:47Z" name: isvc-sklearn-graph-1-predictor.18a6deabb66856cd namespace: kserve-ci-e2e-test resourceVersion: "14733" uid: 182f8491-2755-4574-b626-062a89fccb52 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:05:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: 4f4bcbb0-65e1-4a08-a6d2-0fa549291381 kind: Event lastTimestamp: "2026-04-16T15:05:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:47Z" name: isvc-sklearn-graph-1-predictor.18a6deabb6b91aa6 namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 07389634-862d-4d11-a5e9-0b392eec8c56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:06:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14585" uid: 4f4bcbb0-65e1-4a08-a6d2-0fa549291381 kind: Event lastTimestamp: "2026-04-16T15:07:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:02Z" name: isvc-sklearn-graph-1-predictor.18a6deb2b41f56a3 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: fae2c69b-264f-4294-a4a3-9b9b2c462d7f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:06:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14585" uid: 4f4bcbb0-65e1-4a08-a6d2-0fa549291381 kind: Event lastTimestamp: "2026-04-16T15:07:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:02Z" name: isvc-sklearn-graph-1-predictor.18a6deb2b478e106 namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: 4b656ec8-f400-4bb6-b7c8-d8f985738650 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 5005ac65-2eb8-4393-83de-f2f80b60e7e5 kind: Event lastTimestamp: "2026-04-16T15:06:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:53Z" name: isvc-sklearn-graph-1.18a6dea837b5e202 namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: f9db8fb1-b3c5-4847-b37e-fce7764a695b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 5005ac65-2eb8-4393-83de-f2f80b60e7e5 kind: Event lastTimestamp: "2026-04-16T15:06:53Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-16T15:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:53Z" name: isvc-sklearn-graph-1.18a6debea4402652 namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: d52c3511-a33b-43e2-9991-4e0c58c6dc01 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:05:17.516375Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc namespace: kserve-ci-e2e-test resourceVersion: "14397" uid: a4006d54-acd0-4f3f-a36b-cb21e72133ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-654489c865-9qjkc to ip-10-0-130-229.ec2.internal metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:05:17Z" name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc.18a6dea84fa5d953 namespace: kserve-ci-e2e-test resourceVersion: "14401" uid: 67417980-6cbf-4a60-bab3-00f2a730b219 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc namespace: kserve-ci-e2e-test resourceVersion: "14405" uid: a4006d54-acd0-4f3f-a36b-cb21e72133ff kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:05:17Z" name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc.18a6dea86bcab423 namespace: kserve-ci-e2e-test resourceVersion: "14425" uid: 673b593f-ec30-4fc8-9d88-a7b7671574f0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: a4006d54-acd0-4f3f-a36b-cb21e72133ff kind: Event lastTimestamp: "2026-04-16T15:05:18Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" metadata: creationTimestamp: "2026-04-16T15:05:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:18Z" name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc.18a6dea86d86cc34 namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 344577c3-5cf9-4eb1-aaef-1557eb9692fd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: a4006d54-acd0-4f3f-a36b-cb21e72133ff kind: Event lastTimestamp: "2026-04-16T15:05:29Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" in 11.949s (11.949s including waiting). Image size: 300960724 bytes.' metadata: creationTimestamp: "2026-04-16T15:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:29Z" name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc.18a6deab35c79d50 namespace: kserve-ci-e2e-test resourceVersion: "14525" uid: a6baff05-c42b-42f1-b29d-8f1655b684b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: a4006d54-acd0-4f3f-a36b-cb21e72133ff kind: Event lastTimestamp: "2026-04-16T15:05:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:30Z" name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc.18a6deab3aaaf1a8 namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 70e706b9-8aac-45fb-aa73-3b31b0e7ffce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: a4006d54-acd0-4f3f-a36b-cb21e72133ff kind: Event lastTimestamp: "2026-04-16T15:05:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:30Z" name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc.18a6deab3b4fb55e namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: 954eb583-be84-4093-b277-c2db4c340253 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: a4006d54-acd0-4f3f-a36b-cb21e72133ff kind: Event lastTimestamp: "2026-04-16T15:05:34Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1407" metadata: creationTimestamp: "2026-04-16T15:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:34Z" name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc.18a6deac3a56d3b8 namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 2bcbfe47-5b21-482f-bc49-763c72dc0958 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: a4006d54-acd0-4f3f-a36b-cb21e72133ff kind: Event lastTimestamp: "2026-04-16T15:05:42Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1407" in 7.696s (7.696s including waiting). Image size: 1560613827 bytes.' metadata: creationTimestamp: "2026-04-16T15:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:42Z" name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc.18a6deae051673ad namespace: kserve-ci-e2e-test resourceVersion: "14683" uid: 16ce78fc-7f93-4e56-b124-26d88fdf7a2a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: a4006d54-acd0-4f3f-a36b-cb21e72133ff kind: Event lastTimestamp: "2026-04-16T15:05:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:42Z" name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc.18a6deae08a100da namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: c19c5c5f-d4af-4aab-a451-c0d73c4a216b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: a4006d54-acd0-4f3f-a36b-cb21e72133ff kind: Event lastTimestamp: "2026-04-16T15:05:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:42Z" name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc.18a6deae091be497 namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: b8c8e983-e37c-463d-b6f2-50abfbef493b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T15:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: a4006d54-acd0-4f3f-a36b-cb21e72133ff kind: Event lastTimestamp: "2026-04-16T15:06:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:43Z" name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc.18a6deae190b8bb2 namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: 39ccd086-bfca-467a-974b-1782107b49fe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: a4006d54-acd0-4f3f-a36b-cb21e72133ff kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:27Z" name: isvc-sklearn-graph-2-predictor-654489c865-9qjkc.18a6dec691a61cef namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 5d00c23a-bf7f-4043-a719-14852578b424 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-654489c865 namespace: kserve-ci-e2e-test resourceVersion: "14377" uid: d39f4254-b4c8-4ee7-b6fd-fd190728ff03 kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-654489c865-9qjkc' metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:17Z" name: isvc-sklearn-graph-2-predictor-654489c865.18a6dea84f4b3fcf namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: d5123d5a-3b21-4531-8364-f95df1604cd7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14374" uid: d9b5876b-e259-4788-ad1e-acc4745dda56 kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-654489c865 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:17Z" name: isvc-sklearn-graph-2-predictor.18a6dea8463ac3f3 namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 7a1404d4-0c71-4337-8dc8-b9c857a74018 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:05:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14376" uid: f815af33-d58a-499f-9da7-13cae55cc95a kind: Event lastTimestamp: "2026-04-16T15:05:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:47Z" name: isvc-sklearn-graph-2-predictor.18a6deabc4beb477 namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: 46bf28a1-d904-4e63-af0e-adaef83543cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:05:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14376" uid: f815af33-d58a-499f-9da7-13cae55cc95a kind: Event lastTimestamp: "2026-04-16T15:05:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:47Z" name: isvc-sklearn-graph-2-predictor.18a6deabc50eb18f namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: 8947f987-4da8-4c33-a5bf-a52e2dd919c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:06:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14598" uid: f815af33-d58a-499f-9da7-13cae55cc95a kind: Event lastTimestamp: "2026-04-16T15:07:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:17Z" name: isvc-sklearn-graph-2-predictor.18a6deb2c25237b7 namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: 1688c2ce-a5d7-4946-8c98-7f8928401d04 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:06:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14598" uid: f815af33-d58a-499f-9da7-13cae55cc95a kind: Event lastTimestamp: "2026-04-16T15:07:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:17Z" name: isvc-sklearn-graph-2-predictor.18a6deb2c2aaef5a namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: 13305965-72e4-4896-b18f-38951c3ad041 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14368" uid: 899dff97-4785-4659-81ef-40379d2d6b37 kind: Event lastTimestamp: "2026-04-16T15:06:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:53Z" name: isvc-sklearn-graph-2.18a6dea8460f7ae1 namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: f20ecb47-580d-4c18-9584-d96dead1f08e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15465" uid: 899dff97-4785-4659-81ef-40379d2d6b37 kind: Event lastTimestamp: "2026-04-16T15:06:53Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-16T15:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:53Z" name: isvc-sklearn-graph-2.18a6debea600e7dc namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: 08a907a5-8928-4012-b36c-248d02fd11bc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:05:17.358245Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-5hnph namespace: kserve-ci-e2e-test resourceVersion: "14375" uid: db6ba3de-22ea-4778-9d79-832c269e7aee kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-784cb989b8-5hnph to ip-10-0-130-229.ec2.internal metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:05:17Z" name: isvc-xgboost-graph-predictor-784cb989b8-5hnph.18a6dea846390395 namespace: kserve-ci-e2e-test resourceVersion: "14384" uid: c2fb8837-bfe9-4f63-8798-511986f1a87c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-5hnph namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: db6ba3de-22ea-4778-9d79-832c269e7aee kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:05:17Z" name: isvc-xgboost-graph-predictor-784cb989b8-5hnph.18a6dea86271d2a6 namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: 3eb8fa5b-0881-4a4a-8127-273b69846b49 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-5hnph namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: db6ba3de-22ea-4778-9d79-832c269e7aee kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:17Z" name: isvc-xgboost-graph-predictor-784cb989b8-5hnph.18a6dea863ef954e namespace: kserve-ci-e2e-test resourceVersion: "14421" uid: 4724ab52-1749-42cc-9f9c-2d71681c4f9e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-5hnph namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: db6ba3de-22ea-4778-9d79-832c269e7aee kind: Event lastTimestamp: "2026-04-16T15:05:29Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:c49c84e69ddbb36db782821d4aa3d203f458a00c0c04dd506eb662aae339bb8a" in 12.11s (12.11s including waiting). Image size: 300960724 bytes.' metadata: creationTimestamp: "2026-04-16T15:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:29Z" name: isvc-xgboost-graph-predictor-784cb989b8-5hnph.18a6deab35c8ce3d namespace: kserve-ci-e2e-test resourceVersion: "14526" uid: aaec1485-71af-4367-bad2-39216a2620cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-5hnph namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: db6ba3de-22ea-4778-9d79-832c269e7aee kind: Event lastTimestamp: "2026-04-16T15:05:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:30Z" name: isvc-xgboost-graph-predictor-784cb989b8-5hnph.18a6deab3a9c4410 namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: 79874ac2-9318-4fe1-9a1a-e574c5254147 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-5hnph namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: db6ba3de-22ea-4778-9d79-832c269e7aee kind: Event lastTimestamp: "2026-04-16T15:05:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:30Z" name: isvc-xgboost-graph-predictor-784cb989b8-5hnph.18a6deab3b437cbe namespace: kserve-ci-e2e-test resourceVersion: "14530" uid: ae45ea7b-16d1-4bdd-b5fe-1a20611118bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-5hnph namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: db6ba3de-22ea-4778-9d79-832c269e7aee kind: Event lastTimestamp: "2026-04-16T15:05:34Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-16T15:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:34Z" name: isvc-xgboost-graph-predictor-784cb989b8-5hnph.18a6deac3a463be7 namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: dea39583-cd5a-47bf-9ed4-aa533ab58164 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-5hnph namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: db6ba3de-22ea-4778-9d79-832c269e7aee kind: Event lastTimestamp: "2026-04-16T15:05:52Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.115s (18.115s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-16T15:05:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:52Z" name: isvc-xgboost-graph-predictor-784cb989b8-5hnph.18a6deb0720a8f43 namespace: kserve-ci-e2e-test resourceVersion: "14778" uid: 68cb6928-f1d0-4ece-a11a-776ad693fe9a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-5hnph namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: db6ba3de-22ea-4778-9d79-832c269e7aee kind: Event lastTimestamp: "2026-04-16T15:05:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:05:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:52Z" name: isvc-xgboost-graph-predictor-784cb989b8-5hnph.18a6deb078bff217 namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: aad34c0a-a387-4419-9c50-d070f98c8889 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-5hnph namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: db6ba3de-22ea-4778-9d79-832c269e7aee kind: Event lastTimestamp: "2026-04-16T15:05:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:05:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:52Z" name: isvc-xgboost-graph-predictor-784cb989b8-5hnph.18a6deb07a379bf5 namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: 71f832cb-d30f-4ade-810e-0d3b1d849b44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T15:05:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-5hnph namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: db6ba3de-22ea-4778-9d79-832c269e7aee kind: Event lastTimestamp: "2026-04-16T15:06:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:05:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:44Z" name: isvc-xgboost-graph-predictor-784cb989b8-5hnph.18a6deb0aab8a1a8 namespace: kserve-ci-e2e-test resourceVersion: "15243" uid: 70de9766-e7ab-4415-aa36-31c4906a81f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-5hnph namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: db6ba3de-22ea-4778-9d79-832c269e7aee kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:27Z" name: isvc-xgboost-graph-predictor-784cb989b8-5hnph.18a6dec6892a4e53 namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: d742ec00-f021-4ce6-bff5-eaeaa794bac8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-784cb989b8 namespace: kserve-ci-e2e-test resourceVersion: "14361" uid: 4013d274-9932-4160-adc0-8c8d01c1c9e3 kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: 'Created pod: isvc-xgboost-graph-predictor-784cb989b8-5hnph' metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:17Z" name: isvc-xgboost-graph-predictor-784cb989b8.18a6dea845d354c7 namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: d87231a9-292f-4f5b-8aa5-5bc1e18c0595 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14360" uid: 6be5073e-79a6-4e53-81fe-9c7f5cd2b494 kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: Scaled up replica set isvc-xgboost-graph-predictor-784cb989b8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:17Z" name: isvc-xgboost-graph-predictor.18a6dea841b64ab1 namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: 76a05946-9628-444f-bcb6-5bbf306d7b69 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:05:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14362" uid: 2e76be3d-8951-47aa-b123-d487fdcc0960 kind: Event lastTimestamp: "2026-04-16T15:06:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:02Z" name: isvc-xgboost-graph-predictor.18a6deabc097139c namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: 82c6a40d-f507-43e4-a7f5-10fb63c005a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:05:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14362" uid: 2e76be3d-8951-47aa-b123-d487fdcc0960 kind: Event lastTimestamp: "2026-04-16T15:06:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:02Z" name: isvc-xgboost-graph-predictor.18a6deabc0f6b21f namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: 94255d84-54f7-4e53-b38b-c18e1b3d23f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14593" uid: 2e76be3d-8951-47aa-b123-d487fdcc0960 kind: Event lastTimestamp: "2026-04-16T15:07:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:17Z" name: isvc-xgboost-graph-predictor.18a6deb63ceea9c7 namespace: kserve-ci-e2e-test resourceVersion: "15694" uid: e3d2f98a-ddad-4ee5-8110-fd3719ab5277 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14593" uid: 2e76be3d-8951-47aa-b123-d487fdcc0960 kind: Event lastTimestamp: "2026-04-16T15:07:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:17Z" name: isvc-xgboost-graph-predictor.18a6deb63d4d0fbf namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 92855af5-974e-497f-ae30-5c17e8ea1365 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14342" uid: a466f0e3-4978-47eb-99db-67cc55ba73e7 kind: Event lastTimestamp: "2026-04-16T15:06:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:54Z" name: isvc-xgboost-graph.18a6dea841ecfa0d namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: 2fcfe246-1ec3-4f2b-a3dd-967d7ff42279 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: a466f0e3-4978-47eb-99db-67cc55ba73e7 kind: Event lastTimestamp: "2026-04-16T15:06:54Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-16T15:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:54Z" name: isvc-xgboost-graph.18a6debee24fe263 namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: 0f5cec45-c0ed-4bbd-aa6e-f7804609e923 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:07:16.992766Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-78ccc649-29b6r namespace: kserve-ci-e2e-test resourceVersion: "15673" uid: 5e436b01-46e2-41fc-ac7e-2d4d765f0220 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-78ccc649-29b6r to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:07:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:07:16Z" name: model-chainer-78ccc649-29b6r.18a6dec420fee998 namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 66d252bc-1258-42b2-91ff-e138136ee179 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:17Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-78ccc649-29b6r namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 5e436b01-46e2-41fc-ac7e-2d4d765f0220 kind: Event lastTimestamp: "2026-04-16T15:07:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T15:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:17Z" name: model-chainer-78ccc649-29b6r.18a6dec42876a006 namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: d5e2109c-0424-4302-8bae-0f9bd779ef70 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:18Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-78ccc649-29b6r namespace: kserve-ci-e2e-test resourceVersion: "15682" uid: 5e436b01-46e2-41fc-ac7e-2d4d765f0220 kind: Event lastTimestamp: "2026-04-16T15:07:18Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:07:18Z" name: model-chainer-78ccc649-29b6r.18a6dec45e0e55a9 namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: ec0a6935-3efc-4a52-978a-79b5e90e4d3a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-78ccc649-29b6r namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 5e436b01-46e2-41fc-ac7e-2d4d765f0220 kind: Event lastTimestamp: "2026-04-16T15:07:18Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" already present on machine metadata: creationTimestamp: "2026-04-16T15:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:18Z" name: model-chainer-78ccc649-29b6r.18a6dec45f253ecf namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 4c3b7b7c-2364-4e7f-8c5e-f718c33ebc84 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-78ccc649-29b6r namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 5e436b01-46e2-41fc-ac7e-2d4d765f0220 kind: Event lastTimestamp: "2026-04-16T15:07:18Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-16T15:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:18Z" name: model-chainer-78ccc649-29b6r.18a6dec4627617c2 namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: 1dd78a13-1cdd-4169-b2f5-518dfa71abb7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-78ccc649-29b6r namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 5e436b01-46e2-41fc-ac7e-2d4d765f0220 kind: Event lastTimestamp: "2026-04-16T15:07:18Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-16T15:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:18Z" name: model-chainer-78ccc649-29b6r.18a6dec462e7ee4c namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 3af49754-263d-48f3-bef5-a65fd5168054 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-78ccc649-29b6r namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 5e436b01-46e2-41fc-ac7e-2d4d765f0220 kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:27Z" name: model-chainer-78ccc649-29b6r.18a6dec67b98d6b1 namespace: kserve-ci-e2e-test resourceVersion: "15803" uid: 2071ec2d-d0c6-4ca9-947c-af2e107546ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:07:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-78ccc649-29b6r namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 5e436b01-46e2-41fc-ac7e-2d4d765f0220 kind: Event lastTimestamp: "2026-04-16T15:07:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:54Z" name: model-chainer-78ccc649-29b6r.18a6dec71684f8fd namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 34c545b0-b433-4ebc-8b79-25108482ff6e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-78ccc649 namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: 0984f647-6238-4f03-9f00-20ed3f806802 kind: Event lastTimestamp: "2026-04-16T15:07:16Z" message: 'Created pod: model-chainer-78ccc649-29b6r' metadata: creationTimestamp: "2026-04-16T15:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:16Z" name: model-chainer-78ccc649.18a6dec420a194c3 namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: aa77f2ce-61f6-4375-b13c-903c1310bd2e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: cc8e15cc-f720-4a58-89c8-2e516c4216f6 kind: Event lastTimestamp: "2026-04-16T15:07:16Z" message: Scaled up replica set model-chainer-78ccc649 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:16Z" name: model-chainer.18a6dec41fcb4e8a namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: 73575a85-9283-43a9-a4a5-83299d0bf56e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: f77fa9f8-9fcb-40d4-95ef-b8e08faf91f2 kind: Event lastTimestamp: "2026-04-16T15:07:17Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-16T15:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:07:17Z" name: model-chainer.18a6dec424a9a728 namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 98360eb1-97cf-45d1-aec3-1abfbb6b1f83 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:08:37.334953Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2d44a-d4fd86597-7j4s5 namespace: kserve-ci-e2e-test resourceVersion: "16572" uid: 43d29738-353c-4a39-ac31-5df0a1df037d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-2d44a-d4fd86597-7j4s5 to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:08:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:08:37Z" name: sequence-graph-2d44a-d4fd86597-7j4s5.18a6ded6d5c3672d namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: ab13b84a-64d8-407d-b14a-dcd9d46ea7e3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:37Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2d44a-d4fd86597-7j4s5 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 43d29738-353c-4a39-ac31-5df0a1df037d kind: Event lastTimestamp: "2026-04-16T15:08:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-2d44a-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:37Z" name: sequence-graph-2d44a-d4fd86597-7j4s5.18a6ded6e52cc838 namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: 16f573cb-8f90-4f54-8475-c9629324b572 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:38Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2d44a-d4fd86597-7j4s5 namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: 43d29738-353c-4a39-ac31-5df0a1df037d kind: Event lastTimestamp: "2026-04-16T15:08:38Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:08:38Z" name: sequence-graph-2d44a-d4fd86597-7j4s5.18a6ded712cae867 namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 6968d470-9b6d-452c-a60b-1b61abd7f229 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2d44a} kind: Pod name: sequence-graph-2d44a-d4fd86597-7j4s5 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 43d29738-353c-4a39-ac31-5df0a1df037d kind: Event lastTimestamp: "2026-04-16T15:08:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" already present on machine metadata: creationTimestamp: "2026-04-16T15:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:38Z" name: sequence-graph-2d44a-d4fd86597-7j4s5.18a6ded71405ea04 namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: 4fe3ffd7-ff9d-4ca4-a82e-131fea81a5b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2d44a} kind: Pod name: sequence-graph-2d44a-d4fd86597-7j4s5 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 43d29738-353c-4a39-ac31-5df0a1df037d kind: Event lastTimestamp: "2026-04-16T15:08:38Z" message: 'Created container: sequence-graph-2d44a' metadata: creationTimestamp: "2026-04-16T15:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:38Z" name: sequence-graph-2d44a-d4fd86597-7j4s5.18a6ded71784a36f namespace: kserve-ci-e2e-test resourceVersion: "16601" uid: 24475927-fb3b-44e8-8956-ab8648c34671 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2d44a} kind: Pod name: sequence-graph-2d44a-d4fd86597-7j4s5 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 43d29738-353c-4a39-ac31-5df0a1df037d kind: Event lastTimestamp: "2026-04-16T15:08:38Z" message: Started container sequence-graph-2d44a metadata: creationTimestamp: "2026-04-16T15:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:38Z" name: sequence-graph-2d44a-d4fd86597-7j4s5.18a6ded717f61944 namespace: kserve-ci-e2e-test resourceVersion: "16602" uid: 897c0dfd-5794-4e83-95f9-7a30ed804dce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2d44a} kind: Pod name: sequence-graph-2d44a-d4fd86597-7j4s5 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 43d29738-353c-4a39-ac31-5df0a1df037d kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: Stopping container sequence-graph-2d44a metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:52Z" name: sequence-graph-2d44a-d4fd86597-7j4s5.18a6df4a0476d849 namespace: kserve-ci-e2e-test resourceVersion: "20147" uid: 5a5253b4-cf80-4137-9951-2327be1ca1cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:16:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2d44a} kind: Pod name: sequence-graph-2d44a-d4fd86597-7j4s5 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 43d29738-353c-4a39-ac31-5df0a1df037d kind: Event lastTimestamp: "2026-04-16T15:17:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:19Z" name: sequence-graph-2d44a-d4fd86597-7j4s5.18a6df4ab4647e5d namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: d870dd3d-1c1a-4014-b87e-ca016343ab7e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-2d44a-d4fd86597 namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: bd06002a-89a8-47c6-bdcd-423b0d7bd112 kind: Event lastTimestamp: "2026-04-16T15:08:37Z" message: 'Created pod: sequence-graph-2d44a-d4fd86597-7j4s5' metadata: creationTimestamp: "2026-04-16T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:37Z" name: sequence-graph-2d44a-d4fd86597.18a6ded6d535e6f6 namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: 8ec7fdc9-4bc4-43c3-8987-631c64b0822e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-2d44a namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: da11c703-5916-4ea7-ba1f-39384faecd48 kind: Event lastTimestamp: "2026-04-16T15:08:37Z" message: Scaled up replica set sequence-graph-2d44a-d4fd86597 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:37Z" name: sequence-graph-2d44a.18a6ded6d346d9fd namespace: kserve-ci-e2e-test resourceVersion: "16570" uid: 3599dede-8291-44f0-b181-8dc32251d892 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-2d44a namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: 849839e7-deab-4275-9ac7-7b4abee8d0c1 kind: Event lastTimestamp: "2026-04-16T15:08:37Z" message: InferenceGraph [sequence-graph-2d44a] is Ready metadata: creationTimestamp: "2026-04-16T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:08:37Z" name: sequence-graph-2d44a.18a6ded6d8efe17b namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: 18d2645e-d9ec-438d-9e7f-c4915e5c1fe0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2d44a namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 5e421a9a-f3b6-43e6-bdc3-c267e81cfec1 kind: Event lastTimestamp: "2026-04-16T15:08:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:52Z" name: sequence-graph-2d44a.18a6deda51d283f0 namespace: kserve-ci-e2e-test resourceVersion: "16714" uid: f336512b-297e-4cf0-944e-f6f38cd06426 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2d44a namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 5e421a9a-f3b6-43e6-bdc3-c267e81cfec1 kind: Event lastTimestamp: "2026-04-16T15:08:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:52Z" name: sequence-graph-2d44a.18a6deda522ea00b namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 381f111b-f4f4-4027-aa46-a97453fbf844 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2d44a namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: 5e421a9a-f3b6-43e6-bdc3-c267e81cfec1 kind: Event lastTimestamp: "2026-04-16T15:09:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:09:07Z" name: sequence-graph-2d44a.18a6deddd0cbf012 namespace: kserve-ci-e2e-test resourceVersion: "16817" uid: 32985a4b-58ef-4a8a-8f60-0f0c294c01b3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:09:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2d44a namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: 5e421a9a-f3b6-43e6-bdc3-c267e81cfec1 kind: Event lastTimestamp: "2026-04-16T15:09:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:09:07Z" name: sequence-graph-2d44a.18a6deddd1259c8f namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: 108997af-5fb2-4c75-849f-5250b80a18bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:18:02.209393Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b8642-bbb4f56f-n44lf namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 491a067f-494b-47da-89e3-cc50723f493a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-b8642-bbb4f56f-n44lf to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:18:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:18:02Z" name: sequence-graph-b8642-bbb4f56f-n44lf.18a6df5a5ae752fa namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 819005bf-a975-4062-9edb-0a272a93a5d4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b8642-bbb4f56f-n44lf namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: 491a067f-494b-47da-89e3-cc50723f493a kind: Event lastTimestamp: "2026-04-16T15:18:02Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:18:02Z" name: sequence-graph-b8642-bbb4f56f-n44lf.18a6df5a73dcd3ed namespace: kserve-ci-e2e-test resourceVersion: "21060" uid: 506ccbb1-ff0a-4bd6-8328-beaa4d9ed540 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b8642} kind: Pod name: sequence-graph-b8642-bbb4f56f-n44lf namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 491a067f-494b-47da-89e3-cc50723f493a kind: Event lastTimestamp: "2026-04-16T15:18:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" already present on machine metadata: creationTimestamp: "2026-04-16T15:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:02Z" name: sequence-graph-b8642-bbb4f56f-n44lf.18a6df5a75004df1 namespace: kserve-ci-e2e-test resourceVersion: "21062" uid: daf387a6-c8ea-4fe7-a1aa-6d8b29ff7525 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b8642} kind: Pod name: sequence-graph-b8642-bbb4f56f-n44lf namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 491a067f-494b-47da-89e3-cc50723f493a kind: Event lastTimestamp: "2026-04-16T15:18:02Z" message: 'Created container: sequence-graph-b8642' metadata: creationTimestamp: "2026-04-16T15:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:02Z" name: sequence-graph-b8642-bbb4f56f-n44lf.18a6df5a7820c50c namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: 131d37a1-091a-4a79-bf74-f3d60ec8486c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b8642} kind: Pod name: sequence-graph-b8642-bbb4f56f-n44lf namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 491a067f-494b-47da-89e3-cc50723f493a kind: Event lastTimestamp: "2026-04-16T15:18:02Z" message: Started container sequence-graph-b8642 metadata: creationTimestamp: "2026-04-16T15:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:02Z" name: sequence-graph-b8642-bbb4f56f-n44lf.18a6df5a789104e3 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: d2814b05-9295-4fe6-960d-2ac50cafd7d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b8642} kind: Pod name: sequence-graph-b8642-bbb4f56f-n44lf namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 491a067f-494b-47da-89e3-cc50723f493a kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: Stopping container sequence-graph-b8642 metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:12Z" name: sequence-graph-b8642-bbb4f56f-n44lf.18a6df5cb3e1951b namespace: kserve-ci-e2e-test resourceVersion: "21170" uid: d269184f-7b36-4847-a6f6-7be5f8157d77 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:18:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b8642} kind: Pod name: sequence-graph-b8642-bbb4f56f-n44lf namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 491a067f-494b-47da-89e3-cc50723f493a kind: Event lastTimestamp: "2026-04-16T15:18:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:38Z" name: sequence-graph-b8642-bbb4f56f-n44lf.18a6df5d17ec108e namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: 4d2e71d1-f67c-4c64-8358-28e46df24b70 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-b8642-bbb4f56f namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 7baf8b74-a1f0-42ef-b9a7-1b675c234b34 kind: Event lastTimestamp: "2026-04-16T15:18:02Z" message: 'Created pod: sequence-graph-b8642-bbb4f56f-n44lf' metadata: creationTimestamp: "2026-04-16T15:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:02Z" name: sequence-graph-b8642-bbb4f56f.18a6df5a5a7addd9 namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: 66880196-78c9-4dda-a2e9-b86898bca050 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-b8642 namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 05ada833-8e7e-450a-98d9-23f5793032d5 kind: Event lastTimestamp: "2026-04-16T15:18:02Z" message: Scaled up replica set sequence-graph-b8642-bbb4f56f from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:02Z" name: sequence-graph-b8642.18a6df5a59a7f7fa namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: a6005b0d-0188-4588-8f54-d89ca332623a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-b8642 namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 5b8298ba-f9b0-4a39-86ce-a75de5db0d0b kind: Event lastTimestamp: "2026-04-16T15:18:02Z" message: InferenceGraph [sequence-graph-b8642] is Ready metadata: creationTimestamp: "2026-04-16T15:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:02Z" name: sequence-graph-b8642.18a6df5a5cf2899c namespace: kserve-ci-e2e-test resourceVersion: "21050" uid: 07e7688e-9e8f-4c36-8583-adcaaefa6a0e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-b8642 namespace: kserve-ci-e2e-test resourceVersion: "21024" uid: 5b8298ba-f9b0-4a39-86ce-a75de5db0d0b kind: Event lastTimestamp: "2026-04-16T15:18:02Z" message: 'Failed to update status for InferenceGraph "sequence-graph-b8642": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-b8642": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:02Z" name: sequence-graph-b8642.18a6df5a5f4496d4 namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: b913f862-eeeb-42bb-9a2f-300fe481dc50 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-b8642 namespace: kserve-ci-e2e-test resourceVersion: "21024" uid: 5b8298ba-f9b0-4a39-86ce-a75de5db0d0b kind: Event lastTimestamp: "2026-04-16T15:18:02Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-b8642": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:02Z" name: sequence-graph-b8642.18a6df5a5f44d478 namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 964ff4b3-e505-48c1-9a1c-6575ea8ec2f8 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:19:12.489799Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-cee76-7d9bc88ff6-j7t7v namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: a03a7a5f-6e53-478d-974c-633703f78946 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-cee76-7d9bc88ff6-j7t7v to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:19:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:19:12Z" name: sequence-graph-cee76-7d9bc88ff6-j7t7v.18a6df6ab7f1314d namespace: kserve-ci-e2e-test resourceVersion: "21838" uid: 193405f7-c640-4cf7-8e2a-07ec2a75a36d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:19:12Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-cee76-7d9bc88ff6-j7t7v namespace: kserve-ci-e2e-test resourceVersion: "21835" uid: a03a7a5f-6e53-478d-974c-633703f78946 kind: Event lastTimestamp: "2026-04-16T15:19:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-cee76-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T15:19:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:19:12Z" name: sequence-graph-cee76-7d9bc88ff6-j7t7v.18a6df6ac33d6dc6 namespace: kserve-ci-e2e-test resourceVersion: "21853" uid: 5a1acfea-a8ec-417d-91d0-22e8cbc1952d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:19:13Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-cee76-7d9bc88ff6-j7t7v namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: a03a7a5f-6e53-478d-974c-633703f78946 kind: Event lastTimestamp: "2026-04-16T15:19:13Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:19:13Z" name: sequence-graph-cee76-7d9bc88ff6-j7t7v.18a6df6af4a6b983 namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: cce8323a-db15-4590-a92b-811a06986b2d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:19:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cee76} kind: Pod name: sequence-graph-cee76-7d9bc88ff6-j7t7v namespace: kserve-ci-e2e-test resourceVersion: "21835" uid: a03a7a5f-6e53-478d-974c-633703f78946 kind: Event lastTimestamp: "2026-04-16T15:19:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" already present on machine metadata: creationTimestamp: "2026-04-16T15:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:19:13Z" name: sequence-graph-cee76-7d9bc88ff6-j7t7v.18a6df6af5d9e948 namespace: kserve-ci-e2e-test resourceVersion: "21863" uid: fbc38c2d-dca3-43a4-9ce6-a4871c5b7b52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:19:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cee76} kind: Pod name: sequence-graph-cee76-7d9bc88ff6-j7t7v namespace: kserve-ci-e2e-test resourceVersion: "21835" uid: a03a7a5f-6e53-478d-974c-633703f78946 kind: Event lastTimestamp: "2026-04-16T15:19:13Z" message: 'Created container: sequence-graph-cee76' metadata: creationTimestamp: "2026-04-16T15:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:19:13Z" name: sequence-graph-cee76-7d9bc88ff6-j7t7v.18a6df6af8f382a5 namespace: kserve-ci-e2e-test resourceVersion: "21864" uid: c77d2391-d0da-4765-b833-f3e0aca4942f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:19:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cee76} kind: Pod name: sequence-graph-cee76-7d9bc88ff6-j7t7v namespace: kserve-ci-e2e-test resourceVersion: "21835" uid: a03a7a5f-6e53-478d-974c-633703f78946 kind: Event lastTimestamp: "2026-04-16T15:19:13Z" message: Started container sequence-graph-cee76 metadata: creationTimestamp: "2026-04-16T15:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:19:13Z" name: sequence-graph-cee76-7d9bc88ff6-j7t7v.18a6df6af96385ae namespace: kserve-ci-e2e-test resourceVersion: "21865" uid: 66976dc9-ab0d-499b-b0da-63a20334133a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cee76} kind: Pod name: sequence-graph-cee76-7d9bc88ff6-j7t7v namespace: kserve-ci-e2e-test resourceVersion: "21835" uid: a03a7a5f-6e53-478d-974c-633703f78946 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: Stopping container sequence-graph-cee76 metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:27Z" name: sequence-graph-cee76-7d9bc88ff6-j7t7v.18a6dfdde6590bd7 namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: 321c2628-ccb7-45b4-859b-521f7d61791a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:27:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cee76} kind: Pod name: sequence-graph-cee76-7d9bc88ff6-j7t7v namespace: kserve-ci-e2e-test resourceVersion: "21835" uid: a03a7a5f-6e53-478d-974c-633703f78946 kind: Event lastTimestamp: "2026-04-16T15:27:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:27:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:55Z" name: sequence-graph-cee76-7d9bc88ff6-j7t7v.18a6dfde9b22c1bf namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 6415e6bc-7d44-413a-84c4-67e679d7c0f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:19:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-cee76-7d9bc88ff6 namespace: kserve-ci-e2e-test resourceVersion: "21830" uid: 3786509d-9762-4f89-a9f2-3785c644f3eb kind: Event lastTimestamp: "2026-04-16T15:19:12Z" message: 'Created pod: sequence-graph-cee76-7d9bc88ff6-j7t7v' metadata: creationTimestamp: "2026-04-16T15:19:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:19:12Z" name: sequence-graph-cee76-7d9bc88ff6.18a6df6ab790499c namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 02b6e348-b658-46d8-9594-997d4783e4ee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:19:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: edd55598-3801-4efd-96b2-8f8e7837fab6 kind: Event lastTimestamp: "2026-04-16T15:19:12Z" message: Scaled up replica set sequence-graph-cee76-7d9bc88ff6 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:19:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:19:12Z" name: sequence-graph-cee76.18a6df6ab6eca143 namespace: kserve-ci-e2e-test resourceVersion: "21831" uid: e9d3dcf2-91f9-4edf-ac6b-348120e869f3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:19:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21845" uid: 719888d6-f4d8-4cf8-a26d-3d2e89386728 kind: Event lastTimestamp: "2026-04-16T15:19:12Z" message: InferenceGraph [sequence-graph-cee76] is Ready metadata: creationTimestamp: "2026-04-16T15:19:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:19:12Z" name: sequence-graph-cee76.18a6df6aba6d7145 namespace: kserve-ci-e2e-test resourceVersion: "21847" uid: 5a372441-7bb0-4265-b30e-46f25b9fd88b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:19:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21829" uid: 6efe086f-678d-4498-8906-8045fe96ae28 kind: Event lastTimestamp: "2026-04-16T15:19:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:19:27Z" name: sequence-graph-cee76.18a6df6e35518650 namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: 64df540a-20f2-4436-bd18-e2ea233f13ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:19:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21829" uid: 6efe086f-678d-4498-8906-8045fe96ae28 kind: Event lastTimestamp: "2026-04-16T15:19:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:19:27Z" name: sequence-graph-cee76.18a6df6e35a742bc namespace: kserve-ci-e2e-test resourceVersion: "21971" uid: a436a95b-1bfd-42c4-95c1-6bdfed206870 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:19:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: 6efe086f-678d-4498-8906-8045fe96ae28 kind: Event lastTimestamp: "2026-04-16T15:19:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:19:57Z" name: sequence-graph-cee76.18a6df71b43eca34 namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: 97ad1987-f2ce-4f7a-b2a5-3be810f000dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:19:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: 6efe086f-678d-4498-8906-8045fe96ae28 kind: Event lastTimestamp: "2026-04-16T15:19:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:19:57Z" name: sequence-graph-cee76.18a6df71b49b664f namespace: kserve-ci-e2e-test resourceVersion: "22179" uid: 1a396404-b52e-4204-8dc3-1bbc96dc6c01 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:27:37.597065Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-1fc59-6bbd795f46-rvh7v namespace: kserve-ci-e2e-test resourceVersion: "25930" uid: 7e6bffa8-d6d5-4227-a6a1-66e580dc4f07 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-1fc59-6bbd795f46-rvh7v to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:27:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:27:37Z" name: splitter-graph-1fc59-6bbd795f46-rvh7v.18a6dfe052ae7467 namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 9fa531f6-af01-4dd8-9513-e75113806e58 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:38Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-1fc59-6bbd795f46-rvh7v namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 7e6bffa8-d6d5-4227-a6a1-66e580dc4f07 kind: Event lastTimestamp: "2026-04-16T15:27:38Z" message: Add eth0 [10.134.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:27:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:27:38Z" name: splitter-graph-1fc59-6bbd795f46-rvh7v.18a6dfe06be1c646 namespace: kserve-ci-e2e-test resourceVersion: "25951" uid: 4070cb8f-abfe-44d9-a007-1eaac7d4cced reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1fc59} kind: Pod name: splitter-graph-1fc59-6bbd795f46-rvh7v namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: 7e6bffa8-d6d5-4227-a6a1-66e580dc4f07 kind: Event lastTimestamp: "2026-04-16T15:27:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" already present on machine metadata: creationTimestamp: "2026-04-16T15:27:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:38Z" name: splitter-graph-1fc59-6bbd795f46-rvh7v.18a6dfe06d056d01 namespace: kserve-ci-e2e-test resourceVersion: "25953" uid: 5db3bef9-2292-4b1c-97e8-c52a3ab41f16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1fc59} kind: Pod name: splitter-graph-1fc59-6bbd795f46-rvh7v namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: 7e6bffa8-d6d5-4227-a6a1-66e580dc4f07 kind: Event lastTimestamp: "2026-04-16T15:27:38Z" message: 'Created container: splitter-graph-1fc59' metadata: creationTimestamp: "2026-04-16T15:27:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:38Z" name: splitter-graph-1fc59-6bbd795f46-rvh7v.18a6dfe07027ad4a namespace: kserve-ci-e2e-test resourceVersion: "25954" uid: 427217ae-cba7-4754-8b92-acacd237193d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1fc59} kind: Pod name: splitter-graph-1fc59-6bbd795f46-rvh7v namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: 7e6bffa8-d6d5-4227-a6a1-66e580dc4f07 kind: Event lastTimestamp: "2026-04-16T15:27:38Z" message: Started container splitter-graph-1fc59 metadata: creationTimestamp: "2026-04-16T15:27:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:38Z" name: splitter-graph-1fc59-6bbd795f46-rvh7v.18a6dfe070976355 namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: e2c27bfd-afec-4e1c-bf20-615fae9c2a8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1fc59} kind: Pod name: splitter-graph-1fc59-6bbd795f46-rvh7v namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: 7e6bffa8-d6d5-4227-a6a1-66e580dc4f07 kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: Stopping container splitter-graph-1fc59 metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:47Z" name: splitter-graph-1fc59-6bbd795f46-rvh7v.18a6dfe2ab701c85 namespace: kserve-ci-e2e-test resourceVersion: "26055" uid: cdb5bf9a-e97c-45cf-b026-5b64db406e9e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:27:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1fc59} kind: Pod name: splitter-graph-1fc59-6bbd795f46-rvh7v namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: 7e6bffa8-d6d5-4227-a6a1-66e580dc4f07 kind: Event lastTimestamp: "2026-04-16T15:28:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:27:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:28:14Z" name: splitter-graph-1fc59-6bbd795f46-rvh7v.18a6dfe33336022a namespace: kserve-ci-e2e-test resourceVersion: "26397" uid: b26d3afc-e266-4d6e-a998-bfd1ff2ddbd5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-1fc59-6bbd795f46 namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: c6ce1867-679e-492d-b9e8-0fda613bbfe2 kind: Event lastTimestamp: "2026-04-16T15:27:37Z" message: 'Created pod: splitter-graph-1fc59-6bbd795f46-rvh7v' metadata: creationTimestamp: "2026-04-16T15:27:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:37Z" name: splitter-graph-1fc59-6bbd795f46.18a6dfe05251150e namespace: kserve-ci-e2e-test resourceVersion: "25931" uid: 4a5d440d-9218-4108-91a3-194117ebea39 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-1fc59 namespace: kserve-ci-e2e-test resourceVersion: "25925" uid: 02dc30ef-dfa6-4d3d-8dc7-972e1a595f46 kind: Event lastTimestamp: "2026-04-16T15:27:37Z" message: Scaled up replica set splitter-graph-1fc59-6bbd795f46 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:27:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:37Z" name: splitter-graph-1fc59.18a6dfe051bf10ec namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: 8cc020a1-30c3-4046-9ade-7c4c5fe54b7d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-1fc59 namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 3b8b2029-a5df-4b21-8c15-c0c0186d5957 kind: Event lastTimestamp: "2026-04-16T15:27:37Z" message: InferenceGraph [splitter-graph-1fc59] is Ready metadata: creationTimestamp: "2026-04-16T15:27:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:27:37Z" name: splitter-graph-1fc59.18a6dfe05522c24e namespace: kserve-ci-e2e-test resourceVersion: "25945" uid: c74704f0-dc74-4966-9280-f00e3f138860 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:28:47.882812Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-41c6b-55c4b9fc75-9pkkw namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: 2cd73551-2651-462c-8a61-2f602c2823fe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-41c6b-55c4b9fc75-9pkkw to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:28:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:28:47Z" name: splitter-graph-41c6b-55c4b9fc75-9pkkw.18a6dff0b009d7e9 namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: 16f6cb39-127f-4ab0-9bf4-f5362c086a6b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:48Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-41c6b-55c4b9fc75-9pkkw namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 2cd73551-2651-462c-8a61-2f602c2823fe kind: Event lastTimestamp: "2026-04-16T15:28:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-41c6b-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T15:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:28:48Z" name: splitter-graph-41c6b-55c4b9fc75-9pkkw.18a6dff0b9e10cc4 namespace: kserve-ci-e2e-test resourceVersion: "26759" uid: 2fb080d2-5497-4f87-bbd7-41ca23449061 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:48Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-41c6b-55c4b9fc75-9pkkw namespace: kserve-ci-e2e-test resourceVersion: "26752" uid: 2cd73551-2651-462c-8a61-2f602c2823fe kind: Event lastTimestamp: "2026-04-16T15:28:48Z" message: Add eth0 [10.134.0.55/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:28:48Z" name: splitter-graph-41c6b-55c4b9fc75-9pkkw.18a6dff0eced84bf namespace: kserve-ci-e2e-test resourceVersion: "26769" uid: 205bbcda-77a4-41fd-900a-4cd0c0f55051 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-41c6b} kind: Pod name: splitter-graph-41c6b-55c4b9fc75-9pkkw namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 2cd73551-2651-462c-8a61-2f602c2823fe kind: Event lastTimestamp: "2026-04-16T15:28:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" already present on machine metadata: creationTimestamp: "2026-04-16T15:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:28:48Z" name: splitter-graph-41c6b-55c4b9fc75-9pkkw.18a6dff0ee1c1310 namespace: kserve-ci-e2e-test resourceVersion: "26771" uid: 77a970fb-7c0b-4b6f-983d-d7d843ae3e41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-41c6b} kind: Pod name: splitter-graph-41c6b-55c4b9fc75-9pkkw namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 2cd73551-2651-462c-8a61-2f602c2823fe kind: Event lastTimestamp: "2026-04-16T15:28:48Z" message: 'Created container: splitter-graph-41c6b' metadata: creationTimestamp: "2026-04-16T15:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:28:48Z" name: splitter-graph-41c6b-55c4b9fc75-9pkkw.18a6dff0f119bd5e namespace: kserve-ci-e2e-test resourceVersion: "26772" uid: 98ee3564-6bf0-4f4d-a3ee-4b4874b97ff4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-41c6b} kind: Pod name: splitter-graph-41c6b-55c4b9fc75-9pkkw namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 2cd73551-2651-462c-8a61-2f602c2823fe kind: Event lastTimestamp: "2026-04-16T15:28:48Z" message: Started container splitter-graph-41c6b metadata: creationTimestamp: "2026-04-16T15:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:28:48Z" name: splitter-graph-41c6b-55c4b9fc75-9pkkw.18a6dff0f1899a1b namespace: kserve-ci-e2e-test resourceVersion: "26773" uid: 62b3c96e-8771-412d-8ff3-ffa2b690084a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:37:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-41c6b} kind: Pod name: splitter-graph-41c6b-55c4b9fc75-9pkkw namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 2cd73551-2651-462c-8a61-2f602c2823fe kind: Event lastTimestamp: "2026-04-16T15:37:02Z" message: Stopping container splitter-graph-41c6b metadata: creationTimestamp: "2026-04-16T15:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:37:02Z" name: splitter-graph-41c6b-55c4b9fc75-9pkkw.18a6e063e1a31593 namespace: kserve-ci-e2e-test resourceVersion: "30193" uid: 84441c26-c75e-4b74-983e-431b7055c5d7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-41c6b} kind: Pod name: splitter-graph-41c6b-55c4b9fc75-9pkkw namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 2cd73551-2651-462c-8a61-2f602c2823fe kind: Event lastTimestamp: "2026-04-16T15:37:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:37:30Z" name: splitter-graph-41c6b-55c4b9fc75-9pkkw.18a6e0647ab8cd4e namespace: kserve-ci-e2e-test resourceVersion: "30415" uid: da095acf-54cb-40f6-8356-2330e2b02253 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-41c6b-55c4b9fc75 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: d736b28e-3428-4e97-908d-1c068411cbd2 kind: Event lastTimestamp: "2026-04-16T15:28:47Z" message: 'Created pod: splitter-graph-41c6b-55c4b9fc75-9pkkw' metadata: creationTimestamp: "2026-04-16T15:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:47Z" name: splitter-graph-41c6b-55c4b9fc75.18a6dff0afa64bdc namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: ead2c872-34d1-4c8c-9f64-dfcdfc1e7892 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-41c6b namespace: kserve-ci-e2e-test resourceVersion: "26737" uid: e43cf98e-dc69-4ec7-bfef-38a347b166f2 kind: Event lastTimestamp: "2026-04-16T15:28:47Z" message: Scaled up replica set splitter-graph-41c6b-55c4b9fc75 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:47Z" name: splitter-graph-41c6b.18a6dff0af0e9bef namespace: kserve-ci-e2e-test resourceVersion: "26740" uid: c4aa7fc7-3fb5-49d4-a1c0-81b208700f7b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-41c6b namespace: kserve-ci-e2e-test resourceVersion: "26755" uid: 020ffda0-bc74-41c5-a989-fda98867a06a kind: Event lastTimestamp: "2026-04-16T15:28:47Z" message: InferenceGraph [splitter-graph-41c6b] is Ready metadata: creationTimestamp: "2026-04-16T15:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:28:47Z" name: splitter-graph-41c6b.18a6dff0b2a88545 namespace: kserve-ci-e2e-test resourceVersion: "26757" uid: 21eef2f1-9b0e-437b-9973-e907947a46f9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:29:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-41c6b namespace: kserve-ci-e2e-test resourceVersion: "26738" uid: 02242b79-ea64-4053-943d-532b923afc3b kind: Event lastTimestamp: "2026-04-16T15:29:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:29:17Z" name: splitter-graph-41c6b.18a6dff42d761747 namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: e96bdd1f-9b12-4460-bf4c-87676cf519e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:29:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-41c6b namespace: kserve-ci-e2e-test resourceVersion: "26738" uid: 02242b79-ea64-4053-943d-532b923afc3b kind: Event lastTimestamp: "2026-04-16T15:29:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:29:17Z" name: splitter-graph-41c6b.18a6dff42dd40aad namespace: kserve-ci-e2e-test resourceVersion: "26995" uid: 93797855-c265-4dd1-9b9d-dac198b7a206 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:26:37.532028Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb namespace: kserve-ci-e2e-test resourceVersion: "25243" uid: 25744534-c8ff-42e9-bbd1-7a7752f5acdf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1fc59-predictor-6797966c7-jp9kb to ip-10-0-130-229.ec2.internal metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:26:37Z" name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb.18a6dfd25686b22a namespace: kserve-ci-e2e-test resourceVersion: "25256" uid: 3492cdea-53db-428e-b015-a718ccc2f454 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb namespace: kserve-ci-e2e-test resourceVersion: "25266" uid: 25744534-c8ff-42e9-bbd1-7a7752f5acdf kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:26:37Z" name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb.18a6dfd25d433a36 namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 58684a87-0b6c-4248-ae79-cb118bba9982 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb namespace: kserve-ci-e2e-test resourceVersion: "25250" uid: 25744534-c8ff-42e9-bbd1-7a7752f5acdf kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:26:37Z" name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb.18a6dfd25e5b1433 namespace: kserve-ci-e2e-test resourceVersion: "25284" uid: ac2a824b-076b-4112-a01c-c48b8828b96f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb namespace: kserve-ci-e2e-test resourceVersion: "25250" uid: 25744534-c8ff-42e9-bbd1-7a7752f5acdf kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:26:37Z" name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb.18a6dfd262dd4242 namespace: kserve-ci-e2e-test resourceVersion: "25288" uid: fca27652-aaf3-44ed-807d-8ead3c1b9f85 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb namespace: kserve-ci-e2e-test resourceVersion: "25250" uid: 25744534-c8ff-42e9-bbd1-7a7752f5acdf kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:26:37Z" name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb.18a6dfd26349b833 namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: 40c54797-e011-43ca-b068-ffddf04a3b69 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb namespace: kserve-ci-e2e-test resourceVersion: "25250" uid: 25744534-c8ff-42e9-bbd1-7a7752f5acdf kind: Event lastTimestamp: "2026-04-16T15:27:49Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:49Z" name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb.18a6dfd29da04f8d namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: c16eca03-0bf1-4698-8329-a9b72537376d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb namespace: kserve-ci-e2e-test resourceVersion: "25250" uid: 25744534-c8ff-42e9-bbd1-7a7752f5acdf kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:47Z" name: success-200-isvc-1fc59-predictor-6797966c7-jp9kb.18a6dfe2b12445b7 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: aea6b1a4-60be-48c0-906d-b6c4a32f562b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1fc59-predictor-6797966c7 namespace: kserve-ci-e2e-test resourceVersion: "25221" uid: 89fde4e9-b2c1-4aea-93f1-e5838536b593 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: 'Created pod: success-200-isvc-1fc59-predictor-6797966c7-jp9kb' metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:26:37Z" name: success-200-isvc-1fc59-predictor-6797966c7.18a6dfd256088b61 namespace: kserve-ci-e2e-test resourceVersion: "25247" uid: 50017a7e-670b-4fc8-9e94-396b3cf0baab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1fc59-predictor namespace: kserve-ci-e2e-test resourceVersion: "25214" uid: f19b9382-c0d0-4bf3-95ab-975dc919bae0 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: Scaled up replica set success-200-isvc-1fc59-predictor-6797966c7 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:26:37Z" name: success-200-isvc-1fc59-predictor.18a6dfd252ef3118 namespace: kserve-ci-e2e-test resourceVersion: "25224" uid: ff38eb07-33c1-429d-8323-e3ccfb2cb7a2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1fc59-predictor namespace: kserve-ci-e2e-test resourceVersion: "25220" uid: 9da7d878-483b-41bd-8389-b92511406155 kind: Event lastTimestamp: "2026-04-16T15:26:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:26:52Z" name: success-200-isvc-1fc59-predictor.18a6dfd5d1765871 namespace: kserve-ci-e2e-test resourceVersion: "25431" uid: fd29aecf-3cf9-4a39-961f-297dabc434f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1fc59-predictor namespace: kserve-ci-e2e-test resourceVersion: "25220" uid: 9da7d878-483b-41bd-8389-b92511406155 kind: Event lastTimestamp: "2026-04-16T15:26:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:26:52Z" name: success-200-isvc-1fc59-predictor.18a6dfd5d1d450e1 namespace: kserve-ci-e2e-test resourceVersion: "25433" uid: 9b14a76d-ef27-4a6a-8ae8-623071e78d4f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:27:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1fc59-predictor namespace: kserve-ci-e2e-test resourceVersion: "25432" uid: 9da7d878-483b-41bd-8389-b92511406155 kind: Event lastTimestamp: "2026-04-16T15:27:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:27:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:37Z" name: success-200-isvc-1fc59-predictor.18a6dfd9507f2864 namespace: kserve-ci-e2e-test resourceVersion: "25915" uid: 5464a9ed-58fd-44b7-8ea6-071af4051455 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:27:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1fc59-predictor namespace: kserve-ci-e2e-test resourceVersion: "25432" uid: 9da7d878-483b-41bd-8389-b92511406155 kind: Event lastTimestamp: "2026-04-16T15:27:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:27:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:37Z" name: success-200-isvc-1fc59-predictor.18a6dfd950db41cb namespace: kserve-ci-e2e-test resourceVersion: "25917" uid: ddff014f-43ca-4c52-9852-52a881b81ccc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1fc59 namespace: kserve-ci-e2e-test resourceVersion: "25200" uid: a1ab15ae-6969-44b5-ab97-b3a5850b6433 kind: Event lastTimestamp: "2026-04-16T15:27:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:27:19Z" name: success-200-isvc-1fc59.18a6dfd252d5dbce namespace: kserve-ci-e2e-test resourceVersion: "25644" uid: 5df01a0c-5377-43e7-91ef-ef6828c22c9c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1fc59 namespace: kserve-ci-e2e-test resourceVersion: "25200" uid: a1ab15ae-6969-44b5-ab97-b3a5850b6433 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: 'Failed to update status for InferenceService "success-200-isvc-1fc59": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1fc59": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:26:37Z" name: success-200-isvc-1fc59.18a6dfd25357769b namespace: kserve-ci-e2e-test resourceVersion: "25227" uid: 4f271b71-327b-4f5d-95b2-165c28c69544 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1fc59 namespace: kserve-ci-e2e-test resourceVersion: "25200" uid: a1ab15ae-6969-44b5-ab97-b3a5850b6433 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1fc59": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:26:37Z" name: success-200-isvc-1fc59.18a6dfd25357c348 namespace: kserve-ci-e2e-test resourceVersion: "25229" uid: 610be308-92a5-4b5d-a190-b1c7d63ca333 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1fc59 namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: a1ab15ae-6969-44b5-ab97-b3a5850b6433 kind: Event lastTimestamp: "2026-04-16T15:27:19Z" message: InferenceService [success-200-isvc-1fc59] is Ready metadata: creationTimestamp: "2026-04-16T15:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:27:19Z" name: success-200-isvc-1fc59.18a6dfdc2d415178 namespace: kserve-ci-e2e-test resourceVersion: "25643" uid: 952fdd44-734f-4501-a917-0afb4eb25eb2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:07:27.255098Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 285687a6-4d01-478f-bd88-043e9c2a35a8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2d44a-predictor-595d486c67-sz8xh to ip-10-0-130-229.ec2.internal metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:07:27Z" name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh.18a6dec684adafc9 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 78e4f26e-cd5d-4d35-b359-71aa60904ee5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 285687a6-4d01-478f-bd88-043e9c2a35a8 kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:07:27Z" name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh.18a6dec68c32be62 namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: fca4395c-e301-46f9-bd00-74af3edbbf25 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 285687a6-4d01-478f-bd88-043e9c2a35a8 kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:27Z" name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh.18a6dec68e48e7e5 namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: 8b504711-5995-4ea9-b562-53673ed6ca0f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 285687a6-4d01-478f-bd88-043e9c2a35a8 kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:27Z" name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh.18a6dec69593ea15 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 5a69f56e-89d5-49cc-bbdf-aff612dcefbf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 285687a6-4d01-478f-bd88-043e9c2a35a8 kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:27Z" name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh.18a6dec695f7ce7c namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: 66ef8341-8798-46fa-92df-80dc68acc828 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 285687a6-4d01-478f-bd88-043e9c2a35a8 kind: Event lastTimestamp: "2026-04-16T15:08:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:08:08Z" name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh.18a6dec69ca2ae72 namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: e368666d-2477-4749-9d7e-81fc40b0c4d5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 285687a6-4d01-478f-bd88-043e9c2a35a8 kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:52Z" name: success-200-isvc-2d44a-predictor-595d486c67-sz8xh.18a6df4a09f114cd namespace: kserve-ci-e2e-test resourceVersion: "20176" uid: 221c55bb-a20a-409a-8c5a-e959838b20db reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2d44a-predictor-595d486c67 namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: 8bb0b2a4-3eed-451c-8641-9249d877f5a0 kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: 'Created pod: success-200-isvc-2d44a-predictor-595d486c67-sz8xh' metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:27Z" name: success-200-isvc-2d44a-predictor-595d486c67.18a6dec68457767c namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: 15b472d2-0dd0-45c1-a42a-c6fdef227e3c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2d44a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15816" uid: 70b9794d-be0f-44b1-a0ee-1cc898fdacc9 kind: Event lastTimestamp: "2026-04-16T15:07:27Z" message: Scaled up replica set success-200-isvc-2d44a-predictor-595d486c67 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:27Z" name: success-200-isvc-2d44a-predictor.18a6dec6825eff4e namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: b3252f14-edd3-48d2-8b06-190b448ab9d4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2d44a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: 2353f94b-b41e-4fe2-bb69-b49b038a326e kind: Event lastTimestamp: "2026-04-16T15:07:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:42Z" name: success-200-isvc-2d44a-predictor.18a6deca016af4fc namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: b08430a5-9d94-433c-99b7-2881ec82da29 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2d44a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: 2353f94b-b41e-4fe2-bb69-b49b038a326e kind: Event lastTimestamp: "2026-04-16T15:07:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:42Z" name: success-200-isvc-2d44a-predictor.18a6deca01a7219f namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 8fe5e0e3-a4ea-408f-a3a3-ea7699e84f15 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:07:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2d44a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 2353f94b-b41e-4fe2-bb69-b49b038a326e kind: Event lastTimestamp: "2026-04-16T15:08:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:42Z" name: success-200-isvc-2d44a-predictor.18a6decd80425304 namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: 5eaf6760-fbbe-4544-a69b-6230b199e879 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:07:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2d44a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 2353f94b-b41e-4fe2-bb69-b49b038a326e kind: Event lastTimestamp: "2026-04-16T15:08:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:42Z" name: success-200-isvc-2d44a-predictor.18a6decd80a219cd namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: f709a068-ab40-49ee-8cc6-88447f50e3a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:07:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2d44a namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 2641e2ad-0777-414f-8169-d44217b9915e kind: Event lastTimestamp: "2026-04-16T15:08:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:08:18Z" name: success-200-isvc-2d44a.18a6dec6828ee973 namespace: kserve-ci-e2e-test resourceVersion: "16418" uid: 3c306201-a308-45a2-83bf-7f343a4084be reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2d44a namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 2641e2ad-0777-414f-8169-d44217b9915e kind: Event lastTimestamp: "2026-04-16T15:08:18Z" message: InferenceService [success-200-isvc-2d44a] is Ready metadata: creationTimestamp: "2026-04-16T15:08:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:08:18Z" name: success-200-isvc-2d44a.18a6ded280447111 namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 5747bbe4-9971-45eb-a623-71d9bea551d0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:16:12.331776Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98 namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: f48f0101-4def-4526-9ed7-56889b09bd6f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-36a90-predictor-749c75bdbc-lnz98 to ip-10-0-130-229.ec2.internal metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:16:12Z" name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98.18a6df40c5aff0a8 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: ea5e0de1-31b7-4164-bd13-5f50f22e5689 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98 namespace: kserve-ci-e2e-test resourceVersion: "19799" uid: f48f0101-4def-4526-9ed7-56889b09bd6f kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:16:12Z" name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98.18a6df40cce28ee8 namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: c1be6664-42cd-4e68-ba51-93c59ca62abe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98 namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: f48f0101-4def-4526-9ed7-56889b09bd6f kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:12Z" name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98.18a6df40cdfbe33c namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: d5463432-4558-4e81-a67f-4e508024f861 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98 namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: f48f0101-4def-4526-9ed7-56889b09bd6f kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:12Z" name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98.18a6df40d0f55708 namespace: kserve-ci-e2e-test resourceVersion: "19818" uid: 0f686ac0-7032-4f31-9e28-c513399b2e86 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98 namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: f48f0101-4def-4526-9ed7-56889b09bd6f kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:12Z" name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98.18a6df40d14ce20a namespace: kserve-ci-e2e-test resourceVersion: "19819" uid: f017f719-21bc-4637-b5a5-527f8111e674 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:16:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98 namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: f48f0101-4def-4526-9ed7-56889b09bd6f kind: Event lastTimestamp: "2026-04-16T15:17:24Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:16:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:24Z" name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98.18a6df40f08a4149 namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: 91d16c79-fe77-4c47-80ff-6a294b95b4cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98 namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: f48f0101-4def-4526-9ed7-56889b09bd6f kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:22Z" name: success-200-isvc-36a90-predictor-749c75bdbc-lnz98.18a6df511e449db4 namespace: kserve-ci-e2e-test resourceVersion: "20610" uid: d4246613-1df2-4947-8104-ee2afa9e2225 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-36a90-predictor-749c75bdbc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: d4d57988-8d8e-43a4-a1ca-a717c9cf005d kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: 'Created pod: success-200-isvc-36a90-predictor-749c75bdbc-lnz98' metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:12Z" name: success-200-isvc-36a90-predictor-749c75bdbc.18a6df40c53014f2 namespace: kserve-ci-e2e-test resourceVersion: "19783" uid: dff4152c-4bb0-4d4f-9b98-bc020a5164da reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-36a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: a3493463-4fd0-4705-a97e-0d3614c31f59 kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: Scaled up replica set success-200-isvc-36a90-predictor-749c75bdbc from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:12Z" name: success-200-isvc-36a90-predictor.18a6df40c0592f27 namespace: kserve-ci-e2e-test resourceVersion: "19753" uid: 9e54fbef-f543-4a8a-95c9-2d44052e1ae3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:16:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-36a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: 63708456-5f35-420f-af67-e974006d4f2b kind: Event lastTimestamp: "2026-04-16T15:16:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:42Z" name: success-200-isvc-36a90-predictor.18a6df443ecee2a5 namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: ae528a58-dc13-48e2-8bd7-346333675e9f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:16:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-36a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: 63708456-5f35-420f-af67-e974006d4f2b kind: Event lastTimestamp: "2026-04-16T15:16:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:42Z" name: success-200-isvc-36a90-predictor.18a6df443f284a1c namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: 3d1edb91-8383-4f4b-9b8f-3505a0d43705 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:16:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-36a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 63708456-5f35-420f-af67-e974006d4f2b kind: Event lastTimestamp: "2026-04-16T15:17:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:12Z" name: success-200-isvc-36a90-predictor.18a6df4b3ccc31f0 namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: 97cecd26-bc32-4804-941b-814c2f05562a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:16:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-36a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 63708456-5f35-420f-af67-e974006d4f2b kind: Event lastTimestamp: "2026-04-16T15:17:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:12Z" name: success-200-isvc-36a90-predictor.18a6df4b3d0574a1 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: d7db8950-8f5b-482a-9172-db1600ee5fe4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-36a90 namespace: kserve-ci-e2e-test resourceVersion: "19728" uid: e72104be-b78c-4205-a3ff-a4c8e2a90e8d kind: Event lastTimestamp: "2026-04-16T15:16:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:16:54Z" name: success-200-isvc-36a90.18a6df40c035521a namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: 69bf1d2b-b234-43e1-8b74-224df0123763 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-36a90 namespace: kserve-ci-e2e-test resourceVersion: "20276" uid: e72104be-b78c-4205-a3ff-a4c8e2a90e8d kind: Event lastTimestamp: "2026-04-16T15:16:54Z" message: InferenceService [success-200-isvc-36a90] is Ready metadata: creationTimestamp: "2026-04-16T15:16:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:16:54Z" name: success-200-isvc-36a90.18a6df4a7ffb5332 namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: ddead0a5-64e4-4d40-a0b5-f0c4a46d05b3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:27:47.788240Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8 namespace: kserve-ci-e2e-test resourceVersion: "26087" uid: 10331708-d9ee-4429-ad87-60d96ad8fb70 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8 to ip-10-0-130-229.ec2.internal metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:27:47Z" name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8.18a6dfe2b21f6e45 namespace: kserve-ci-e2e-test resourceVersion: "26092" uid: 1b1de61e-0488-43cf-a85b-4ccf3d9c4ea3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8 namespace: kserve-ci-e2e-test resourceVersion: "26097" uid: 10331708-d9ee-4429-ad87-60d96ad8fb70 kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:27:47Z" name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8.18a6dfe2b9abdafb namespace: kserve-ci-e2e-test resourceVersion: "26139" uid: 68ae9f76-5a36-495a-b445-b883fa43d08b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8 namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 10331708-d9ee-4429-ad87-60d96ad8fb70 kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:47Z" name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8.18a6dfe2bae52f13 namespace: kserve-ci-e2e-test resourceVersion: "26143" uid: 5d1f47d2-10b4-4160-9348-2f8bf34bf603 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8 namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 10331708-d9ee-4429-ad87-60d96ad8fb70 kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:47Z" name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8.18a6dfe2bdbc9578 namespace: kserve-ci-e2e-test resourceVersion: "26144" uid: e7eaf40e-a6ea-4980-b9dd-d899476fdb54 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8 namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 10331708-d9ee-4429-ad87-60d96ad8fb70 kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:47Z" name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8.18a6dfe2be2513cf namespace: kserve-ci-e2e-test resourceVersion: "26147" uid: 6450780e-56e8-4ee2-8595-bbd49396de1d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:27:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8 namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 10331708-d9ee-4429-ad87-60d96ad8fb70 kind: Event lastTimestamp: "2026-04-16T15:28:19Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:27:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:28:19Z" name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8.18a6dfe2f5445952 namespace: kserve-ci-e2e-test resourceVersion: "26459" uid: afd0bc75-f4d3-4406-85e5-650fc9ca7ef8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:37:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8 namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 10331708-d9ee-4429-ad87-60d96ad8fb70 kind: Event lastTimestamp: "2026-04-16T15:37:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:37:02Z" name: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8.18a6e063e619f0b2 namespace: kserve-ci-e2e-test resourceVersion: "30206" uid: 8c65c5be-81b4-4d1a-ac70-22b5260027dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-41c6b-predictor-86f5c7cb45 namespace: kserve-ci-e2e-test resourceVersion: "26075" uid: 5e529461-a58e-4db2-a979-260d2c56b6cc kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: 'Created pod: success-200-isvc-41c6b-predictor-86f5c7cb45-9h6d8' metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:47Z" name: success-200-isvc-41c6b-predictor-86f5c7cb45.18a6dfe2b1a7a1ce namespace: kserve-ci-e2e-test resourceVersion: "26088" uid: 2da6e9a4-2a1b-46ed-85c2-63110a736086 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-41c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 7a5d355f-b1d8-4f61-9789-638cc3c0e72f kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: Scaled up replica set success-200-isvc-41c6b-predictor-86f5c7cb45 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:47Z" name: success-200-isvc-41c6b-predictor.18a6dfe2b048e27c namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: ab65bdbe-2813-435a-9bce-06edc6914f5a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:28:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-41c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26077" uid: 4bd24150-569d-41bd-b73d-aad7b86a4a38 kind: Event lastTimestamp: "2026-04-16T15:28:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:17Z" name: success-200-isvc-41c6b-predictor.18a6dfe62f0a28cf namespace: kserve-ci-e2e-test resourceVersion: "26417" uid: 000de0f4-8ff1-41fb-a247-847d7060340b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:28:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-41c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26077" uid: 4bd24150-569d-41bd-b73d-aad7b86a4a38 kind: Event lastTimestamp: "2026-04-16T15:28:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:17Z" name: success-200-isvc-41c6b-predictor.18a6dfe62f7bc427 namespace: kserve-ci-e2e-test resourceVersion: "26418" uid: db525eba-c80d-4bbd-ac2f-f84f2dae24f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:28:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-41c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26301" uid: 4bd24150-569d-41bd-b73d-aad7b86a4a38 kind: Event lastTimestamp: "2026-04-16T15:28:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:28:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:47Z" name: success-200-isvc-41c6b-predictor.18a6dfed2d08e26a namespace: kserve-ci-e2e-test resourceVersion: "26727" uid: 756e0fb7-da3b-403a-b1af-558fa6a9b97f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:28:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-41c6b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26301" uid: 4bd24150-569d-41bd-b73d-aad7b86a4a38 kind: Event lastTimestamp: "2026-04-16T15:28:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:28:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:47Z" name: success-200-isvc-41c6b-predictor.18a6dfed2d7890a1 namespace: kserve-ci-e2e-test resourceVersion: "26730" uid: 62effaa8-9a1c-4a53-8df6-f79a88469266 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-41c6b namespace: kserve-ci-e2e-test resourceVersion: "26058" uid: cb40bad1-5d97-4029-a1b6-17f553c7240b kind: Event lastTimestamp: "2026-04-16T15:28:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:28:29Z" name: success-200-isvc-41c6b.18a6dfe2b05a41ed namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: 3dcc2d6b-6146-48c1-9bb7-2a7a0367a3e1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-41c6b namespace: kserve-ci-e2e-test resourceVersion: "26058" uid: cb40bad1-5d97-4029-a1b6-17f553c7240b kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: 'Failed to update status for InferenceService "success-200-isvc-41c6b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-41c6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:27:47Z" name: success-200-isvc-41c6b.18a6dfe2b0e3eb71 namespace: kserve-ci-e2e-test resourceVersion: "26083" uid: 0ef89756-29f2-4e04-b392-459493741941 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-41c6b namespace: kserve-ci-e2e-test resourceVersion: "26058" uid: cb40bad1-5d97-4029-a1b6-17f553c7240b kind: Event lastTimestamp: "2026-04-16T15:27:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-41c6b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:27:47Z" name: success-200-isvc-41c6b.18a6dfe2b0e43bab namespace: kserve-ci-e2e-test resourceVersion: "26085" uid: c71dea58-f9e3-43fb-b873-4ddcd63c4ede reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-41c6b namespace: kserve-ci-e2e-test resourceVersion: "26537" uid: cb40bad1-5d97-4029-a1b6-17f553c7240b kind: Event lastTimestamp: "2026-04-16T15:28:29Z" message: InferenceService [success-200-isvc-41c6b] is Ready metadata: creationTimestamp: "2026-04-16T15:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:28:29Z" name: success-200-isvc-41c6b.18a6dfec84a3542e namespace: kserve-ci-e2e-test resourceVersion: "26538" uid: bfefdedc-c888-43dc-aaf5-8805e03a1ff2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:27:27.316093Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs namespace: kserve-ci-e2e-test resourceVersion: "25755" uid: 6911b433-b46c-4c3f-8e79-91159bcdd409 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs to ip-10-0-130-229.ec2.internal metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:27:27Z" name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs.18a6dfddede33e2e namespace: kserve-ci-e2e-test resourceVersion: "25760" uid: 9586eb08-2604-4a39-9914-23d7ba027fbc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs namespace: kserve-ci-e2e-test resourceVersion: "25768" uid: 6911b433-b46c-4c3f-8e79-91159bcdd409 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:27:27Z" name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs.18a6dfddf516762d namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: b028059e-c1dd-493f-9c76-383c6daf3fae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs namespace: kserve-ci-e2e-test resourceVersion: "25757" uid: 6911b433-b46c-4c3f-8e79-91159bcdd409 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:27Z" name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs.18a6dfddf62407bb namespace: kserve-ci-e2e-test resourceVersion: "25808" uid: 185cb4c7-89c6-4627-8bdc-8174e98e48fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs namespace: kserve-ci-e2e-test resourceVersion: "25757" uid: 6911b433-b46c-4c3f-8e79-91159bcdd409 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:27Z" name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs.18a6dfddfb73a2cc namespace: kserve-ci-e2e-test resourceVersion: "25812" uid: a700cc95-5551-4ff1-b232-62b4b9b3f0f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs namespace: kserve-ci-e2e-test resourceVersion: "25757" uid: 6911b433-b46c-4c3f-8e79-91159bcdd409 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:27Z" name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs.18a6dfddfc2683e5 namespace: kserve-ci-e2e-test resourceVersion: "25814" uid: d3b17d6a-bad3-4320-bc1d-4ddd8939f39a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs namespace: kserve-ci-e2e-test resourceVersion: "25757" uid: 6911b433-b46c-4c3f-8e79-91159bcdd409 kind: Event lastTimestamp: "2026-04-16T15:44:58Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:44:58Z" name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs.18a6dfde0df074c8 namespace: kserve-ci-e2e-test resourceVersion: "33543" uid: a2256d79-7d28-402b-9183-6f3db7494f1b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:44:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs namespace: kserve-ci-e2e-test resourceVersion: "25757" uid: 6911b433-b46c-4c3f-8e79-91159bcdd409 kind: Event lastTimestamp: "2026-04-16T15:44:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:44:56Z" name: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs.18a6e0d24e50b244 namespace: kserve-ci-e2e-test resourceVersion: "33483" uid: 43512c43-108d-4bc9-94d8-40d04c7b7538 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9c7d6-predictor-6d4df94467 namespace: kserve-ci-e2e-test resourceVersion: "25740" uid: 43bd546f-d877-4871-b503-90626213d986 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: 'Created pod: success-200-isvc-9c7d6-predictor-6d4df94467-hc4fs' metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:27Z" name: success-200-isvc-9c7d6-predictor-6d4df94467.18a6dfdded7beeb6 namespace: kserve-ci-e2e-test resourceVersion: "25756" uid: dd9d5d82-165a-485c-b21b-07ea70524b48 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9c7d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 75701284-1996-4de4-a810-45580cca5eec kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: Scaled up replica set success-200-isvc-9c7d6-predictor-6d4df94467 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:27Z" name: success-200-isvc-9c7d6-predictor.18a6dfddebc5e3df namespace: kserve-ci-e2e-test resourceVersion: "25746" uid: 33aeb9ac-c972-40da-bdd8-ee0d9b52ee86 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c7d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25739" uid: 52990086-325a-4603-bef6-30d74fa2df20 kind: Event lastTimestamp: "2026-04-16T15:27:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:42Z" name: success-200-isvc-9c7d6-predictor.18a6dfe16ab73ec0 namespace: kserve-ci-e2e-test resourceVersion: "25992" uid: b4695bf6-268f-4c93-b46e-08628adc6ea7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c7d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25739" uid: 52990086-325a-4603-bef6-30d74fa2df20 kind: Event lastTimestamp: "2026-04-16T15:27:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:27:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:27:42Z" name: success-200-isvc-9c7d6-predictor.18a6dfe16b10f52a namespace: kserve-ci-e2e-test resourceVersion: "25994" uid: 00e45f96-5dd4-4f8a-b088-f020f328f881 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:27:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c7d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: 52990086-325a-4603-bef6-30d74fa2df20 kind: Event lastTimestamp: "2026-04-16T15:28:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:27:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:42Z" name: success-200-isvc-9c7d6-predictor.18a6dfe4e9cb0f21 namespace: kserve-ci-e2e-test resourceVersion: "26680" uid: abf34a29-4379-4d1b-99c2-0adedcccbb07 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:27:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c7d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: 52990086-325a-4603-bef6-30d74fa2df20 kind: Event lastTimestamp: "2026-04-16T15:28:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:27:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:42Z" name: success-200-isvc-9c7d6-predictor.18a6dfe4ea306a4f namespace: kserve-ci-e2e-test resourceVersion: "26681" uid: 3f62423c-e941-49fc-818e-1ee02b496fc0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "25727" uid: 45d4607c-2b8e-4347-8ae7-91bca87db677 kind: Event lastTimestamp: "2026-04-16T15:28:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:28:18Z" name: success-200-isvc-9c7d6.18a6dfddebce2591 namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: 9c84616f-9a73-4d06-8e4e-316f7c484abe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "25727" uid: 45d4607c-2b8e-4347-8ae7-91bca87db677 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: 'Failed to update status for InferenceService "success-200-isvc-9c7d6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9c7d6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:27:27Z" name: success-200-isvc-9c7d6.18a6dfddec703a90 namespace: kserve-ci-e2e-test resourceVersion: "25752" uid: 0a09f783-4d1c-4ada-8e1c-bc2c45903ce5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "25727" uid: 45d4607c-2b8e-4347-8ae7-91bca87db677 kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9c7d6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:27:27Z" name: success-200-isvc-9c7d6.18a6dfddec708d1c namespace: kserve-ci-e2e-test resourceVersion: "25753" uid: 564d51c1-0f79-41d9-94ca-997128566292 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "26440" uid: 45d4607c-2b8e-4347-8ae7-91bca87db677 kind: Event lastTimestamp: "2026-04-16T15:28:18Z" message: InferenceService [success-200-isvc-9c7d6] is Ready metadata: creationTimestamp: "2026-04-16T15:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:28:18Z" name: success-200-isvc-9c7d6.18a6dfe9f19f1f16 namespace: kserve-ci-e2e-test resourceVersion: "26442" uid: 15502c74-c072-49d9-a7f3-b663e5ba77fa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:17:22.557356Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: 2c987b34-a142-4636-a350-24394176c8a4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b2be2-predictor-855dffb447-hvjpx to ip-10-0-130-229.ec2.internal metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:17:22Z" name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx.18a6df511f754924 namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: bc63ddab-1924-4073-ba72-63892a04d64c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx namespace: kserve-ci-e2e-test resourceVersion: "20625" uid: 2c987b34-a142-4636-a350-24394176c8a4 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:17:22Z" name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx.18a6df5126b32e54 namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 861951b6-128f-40b0-a52e-5e6ba30349cc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 2c987b34-a142-4636-a350-24394176c8a4 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:22Z" name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx.18a6df5128041bea namespace: kserve-ci-e2e-test resourceVersion: "20669" uid: 55c44d04-bb06-4e0f-9e76-cf1c7c3bbc0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 2c987b34-a142-4636-a350-24394176c8a4 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:22Z" name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx.18a6df512af0c104 namespace: kserve-ci-e2e-test resourceVersion: "20670" uid: dba116c5-be0a-49f4-819c-78c5d4b3d9ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 2c987b34-a142-4636-a350-24394176c8a4 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:22Z" name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx.18a6df512b532a78 namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: c9cc461e-2f41-4e12-92a8-c1d7b727f567 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:17:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 2c987b34-a142-4636-a350-24394176c8a4 kind: Event lastTimestamp: "2026-04-16T15:17:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:17:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:17:54Z" name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx.18a6df5148c38d05 namespace: kserve-ci-e2e-test resourceVersion: "20969" uid: e1366f79-0346-43df-8882-0f232dfcee8b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 2c987b34-a142-4636-a350-24394176c8a4 kind: Event lastTimestamp: "2026-04-16T15:26:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:26:37Z" name: success-200-isvc-b2be2-predictor-855dffb447-hvjpx.18a6dfd253499807 namespace: kserve-ci-e2e-test resourceVersion: "25226" uid: 5c33e072-c602-4f92-8292-5bb300ffa006 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b2be2-predictor-855dffb447 namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: 20b64e97-f90a-45fe-be6d-9723af7abfc8 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: 'Created pod: success-200-isvc-b2be2-predictor-855dffb447-hvjpx' metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:22Z" name: success-200-isvc-b2be2-predictor-855dffb447.18a6df511eed24e9 namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: e51a17f1-4c92-4a09-b646-d89c689f6795 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b2be2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: a0b6365c-e2a5-4ad8-9dfe-93e233372a32 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: Scaled up replica set success-200-isvc-b2be2-predictor-855dffb447 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:22Z" name: success-200-isvc-b2be2-predictor.18a6df511db50d86 namespace: kserve-ci-e2e-test resourceVersion: "20604" uid: 8d39172b-0792-4d4e-b50f-e7261d7104be reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:17:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2be2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: 356a4ead-c434-49fd-8d34-dd3531023105 kind: Event lastTimestamp: "2026-04-16T15:18:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:07Z" name: success-200-isvc-b2be2-predictor.18a6df549c29646d namespace: kserve-ci-e2e-test resourceVersion: "21114" uid: 5613abf8-3279-4dcb-8ae5-314ceb5a8c46 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T15:17:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2be2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: 356a4ead-c434-49fd-8d34-dd3531023105 kind: Event lastTimestamp: "2026-04-16T15:18:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:07Z" name: success-200-isvc-b2be2-predictor.18a6df549ca555d1 namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: fa920f53-741b-421a-9642-c25ca6f0b43e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:18:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2be2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: 356a4ead-c434-49fd-8d34-dd3531023105 kind: Event lastTimestamp: "2026-04-16T15:18:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:18:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:37Z" name: success-200-isvc-b2be2-predictor.18a6df5f18b3cc5a namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: a191eb64-7ceb-430e-8afb-6430a24fa678 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:18:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2be2-predictor namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: 356a4ead-c434-49fd-8d34-dd3531023105 kind: Event lastTimestamp: "2026-04-16T15:18:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:18:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:37Z" name: success-200-isvc-b2be2-predictor.18a6df5f19223030 namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: cd1f1a45-56f5-46bc-aa9a-fd4c8c9fc03d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: 8d45f7f7-246b-4f5a-bd59-afe879960a55 kind: Event lastTimestamp: "2026-04-16T15:18:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:04Z" name: success-200-isvc-b2be2.18a6df511d8f33b3 namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: ad01db9f-b9f7-4fd5-b302-19f54e9b5f9b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: 8d45f7f7-246b-4f5a-bd59-afe879960a55 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: 'Failed to update status for InferenceService "success-200-isvc-b2be2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b2be2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:17:22Z" name: success-200-isvc-b2be2.18a6df511de26cd4 namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: 487a033f-faaa-4d7f-9f86-e4b8de1562f5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: 8d45f7f7-246b-4f5a-bd59-afe879960a55 kind: Event lastTimestamp: "2026-04-16T15:17:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b2be2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:17:22Z" name: success-200-isvc-b2be2.18a6df511de2a927 namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 45b59518-d95a-4b5f-a713-9e4b7c10bb40 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2be2 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: 8d45f7f7-246b-4f5a-bd59-afe879960a55 kind: Event lastTimestamp: "2026-04-16T15:18:04Z" message: InferenceService [success-200-isvc-b2be2] is Ready metadata: creationTimestamp: "2026-04-16T15:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:04Z" name: success-200-isvc-b2be2.18a6df5ad85ac14f namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: 19dd0abf-dfc3-4140-b2f2-547323e8dcbf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:16:52.150590Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b8642-predictor-89bc8697-8tq5g namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: 7661a2c7-d428-401d-8a13-ea94d983720c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b8642-predictor-89bc8697-8tq5g to ip-10-0-130-229.ec2.internal metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:16:52Z" name: success-200-isvc-b8642-predictor-89bc8697-8tq5g.18a6df4a0b12d785 namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: c83c838d-c2ae-4504-8363-0dbfebb33848 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b8642-predictor-89bc8697-8tq5g namespace: kserve-ci-e2e-test resourceVersion: "20189" uid: 7661a2c7-d428-401d-8a13-ea94d983720c kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:16:52Z" name: success-200-isvc-b8642-predictor-89bc8697-8tq5g.18a6df4a1232f688 namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: 8a99ccb2-c91c-4164-9742-be7b4b4d9284 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b8642-predictor-89bc8697-8tq5g namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 7661a2c7-d428-401d-8a13-ea94d983720c kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:52Z" name: success-200-isvc-b8642-predictor-89bc8697-8tq5g.18a6df4a13780373 namespace: kserve-ci-e2e-test resourceVersion: "20232" uid: e62464e1-0399-415e-a390-42e42f970278 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b8642-predictor-89bc8697-8tq5g namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 7661a2c7-d428-401d-8a13-ea94d983720c kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:52Z" name: success-200-isvc-b8642-predictor-89bc8697-8tq5g.18a6df4a16136dc6 namespace: kserve-ci-e2e-test resourceVersion: "20234" uid: 661dcc74-95bb-4547-a8f6-d7270259f420 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b8642-predictor-89bc8697-8tq5g namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 7661a2c7-d428-401d-8a13-ea94d983720c kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:52Z" name: success-200-isvc-b8642-predictor-89bc8697-8tq5g.18a6df4a166c7d9c namespace: kserve-ci-e2e-test resourceVersion: "20235" uid: 5c67a371-7151-4830-b857-0b1c91bf28a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T15:16:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b8642-predictor-89bc8697-8tq5g namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 7661a2c7-d428-401d-8a13-ea94d983720c kind: Event lastTimestamp: "2026-04-16T15:18:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:16:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:14Z" name: success-200-isvc-b8642-predictor-89bc8697-8tq5g.18a6df4a4754669c namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 7c6da20c-b94b-4c26-850c-7bbd35a93529 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b8642-predictor-89bc8697-8tq5g namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 7661a2c7-d428-401d-8a13-ea94d983720c kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:12Z" name: success-200-isvc-b8642-predictor-89bc8697-8tq5g.18a6df5cbad40c5f namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: 717a6faf-48a2-4196-be9f-a6c6647a9f6a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b8642-predictor-89bc8697 namespace: kserve-ci-e2e-test resourceVersion: "20169" uid: a560574c-27a1-4e61-9adf-fb592b1ccad6 kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: 'Created pod: success-200-isvc-b8642-predictor-89bc8697-8tq5g' metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:52Z" name: success-200-isvc-b8642-predictor-89bc8697.18a6df4a0a9708f0 namespace: kserve-ci-e2e-test resourceVersion: "20180" uid: 6e4f285a-a86e-4087-a5b6-e75be8da3c86 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b8642-predictor namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: a1cb82bf-2b3d-4aa4-843f-2be75c19736a kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: Scaled up replica set success-200-isvc-b8642-predictor-89bc8697 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:16:52Z" name: success-200-isvc-b8642-predictor.18a6df4a094c7836 namespace: kserve-ci-e2e-test resourceVersion: "20171" uid: 093d22ff-1b9e-42ca-b031-2a987b5748ad reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b8642-predictor namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 11402839-d5b9-4c99-92c3-0821daa85472 kind: Event lastTimestamp: "2026-04-16T15:17:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:07Z" name: success-200-isvc-b8642-predictor.18a6df4d87f96901 namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: ea57f124-1509-4aa8-8c3e-03acb0a01976 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b8642-predictor namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: 11402839-d5b9-4c99-92c3-0821daa85472 kind: Event lastTimestamp: "2026-04-16T15:17:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:17:07Z" name: success-200-isvc-b8642-predictor.18a6df4d8854470d namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 4aad5944-de0d-4ea9-b8f5-da4d20b5cff2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b8642-predictor namespace: kserve-ci-e2e-test resourceVersion: "20394" uid: 11402839-d5b9-4c99-92c3-0821daa85472 kind: Event lastTimestamp: "2026-04-16T15:18:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:07Z" name: success-200-isvc-b8642-predictor.18a6df5106f80524 namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 0cf23913-bec0-4783-b3e1-70ed2ae8f822 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:17:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b8642-predictor namespace: kserve-ci-e2e-test resourceVersion: "20394" uid: 11402839-d5b9-4c99-92c3-0821daa85472 kind: Event lastTimestamp: "2026-04-16T15:18:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:17:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:07Z" name: success-200-isvc-b8642-predictor.18a6df5107539af1 namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: 0d256027-75dc-45f1-886c-757b3c5ae46f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b8642 namespace: kserve-ci-e2e-test resourceVersion: "20150" uid: a4e1afdb-71de-4ef5-bb0a-ed00d5d27652 kind: Event lastTimestamp: "2026-04-16T15:17:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:17:44Z" name: success-200-isvc-b8642.18a6df4a0957206f namespace: kserve-ci-e2e-test resourceVersion: "20885" uid: f9a49cab-5179-4d4a-bfd6-73ebe8eac42a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b8642 namespace: kserve-ci-e2e-test resourceVersion: "20150" uid: a4e1afdb-71de-4ef5-bb0a-ed00d5d27652 kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: 'Failed to update status for InferenceService "success-200-isvc-b8642": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b8642": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:16:52Z" name: success-200-isvc-b8642.18a6df4a09d81da2 namespace: kserve-ci-e2e-test resourceVersion: "20175" uid: ca90c7d5-a822-4c74-a58b-751c8287e0c4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b8642 namespace: kserve-ci-e2e-test resourceVersion: "20150" uid: a4e1afdb-71de-4ef5-bb0a-ed00d5d27652 kind: Event lastTimestamp: "2026-04-16T15:16:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b8642": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:16:52Z" name: success-200-isvc-b8642.18a6df4a09d87eba namespace: kserve-ci-e2e-test resourceVersion: "20178" uid: ca0dc101-ef44-4737-ba5c-418e081d78e0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:17:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b8642 namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: a4e1afdb-71de-4ef5-bb0a-ed00d5d27652 kind: Event lastTimestamp: "2026-04-16T15:17:44Z" message: InferenceService [success-200-isvc-b8642] is Ready metadata: creationTimestamp: "2026-04-16T15:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:17:44Z" name: success-200-isvc-b8642.18a6df562abacdab namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 447ee90e-2eb7-4ec0-8345-d1b053223cb5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:06:47.355066Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: c1c7f898-5967-469f-be6b-a239528e1b81 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf to ip-10-0-130-229.ec2.internal metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:06:47Z" name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf.18a6debd3a7385ae namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: 73847a7e-a730-4b66-8154-954f3ef07f96 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: c1c7f898-5967-469f-be6b-a239528e1b81 kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:06:47Z" name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf.18a6debd410e9c1e namespace: kserve-ci-e2e-test resourceVersion: "15375" uid: 930c9120-beea-4a53-9ce0-9db5c981cfd3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: c1c7f898-5967-469f-be6b-a239528e1b81 kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:47Z" name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf.18a6debd4231d252 namespace: kserve-ci-e2e-test resourceVersion: "15377" uid: cd01cb04-779f-4399-90ab-34b091e36e7b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: c1c7f898-5967-469f-be6b-a239528e1b81 kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:47Z" name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf.18a6debd45459a21 namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: 187391dc-0d12-41e8-9f73-01a10e861403 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: c1c7f898-5967-469f-be6b-a239528e1b81 kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:47Z" name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf.18a6debd45a19498 namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: 38af4c82-4933-41e0-ab54-831b6dabc3b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: c1c7f898-5967-469f-be6b-a239528e1b81 kind: Event lastTimestamp: "2026-04-16T15:07:28Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:28Z" name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf.18a6debd462049df namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 3b85e95c-2fe1-4f70-aa8c-58a897320d33 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: c1c7f898-5967-469f-be6b-a239528e1b81 kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:12Z" name: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf.18a6df40bfe02165 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 4387b816-6097-4313-8b02-9bbe43fe7ba6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c9f8b-predictor-b9f4c48f7 namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: cfc6e804-c074-41bd-9883-5c98b7b1eea3 kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: 'Created pod: success-200-isvc-c9f8b-predictor-b9f4c48f7-z6skf' metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:47Z" name: success-200-isvc-c9f8b-predictor-b9f4c48f7.18a6debd3a13ea5f namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: 33ed8502-4ead-44a3-aca8-2cd7b7cd4ae8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c9f8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 276166c9-9c87-4c41-b18e-736af5f838ec kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: Scaled up replica set success-200-isvc-c9f8b-predictor-b9f4c48f7 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:47Z" name: success-200-isvc-c9f8b-predictor.18a6debd38ba0060 namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: a7ce5afe-227e-448a-9c31-9cbe30d889e8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:07:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c9f8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: c2650b9e-eec5-467c-aec3-ab3169319efc kind: Event lastTimestamp: "2026-04-16T15:08:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:02Z" name: success-200-isvc-c9f8b-predictor.18a6dec0b72afd13 namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 26928973-0976-4315-ad28-1ecc44002a02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:07:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c9f8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: c2650b9e-eec5-467c-aec3-ab3169319efc kind: Event lastTimestamp: "2026-04-16T15:08:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:02Z" name: success-200-isvc-c9f8b-predictor.18a6dec0b788f492 namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: f6eb1ed6-013d-4e36-826e-3cc3479a1550 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c9f8b namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: fa721402-edb8-44ed-98ab-7a1e1cbb9286 kind: Event lastTimestamp: "2026-04-16T15:07:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:07:38Z" name: success-200-isvc-c9f8b.18a6debd3898b7b3 namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 9cac1051-0e2e-403b-9f1b-11fcc304b8c5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c9f8b namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: fa721402-edb8-44ed-98ab-7a1e1cbb9286 kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: 'Failed to update status for InferenceService "success-200-isvc-c9f8b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c9f8b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:47Z" name: success-200-isvc-c9f8b.18a6debd38e06fc0 namespace: kserve-ci-e2e-test resourceVersion: "15332" uid: e1914a80-a8a2-4b8a-afa6-321313df5f4e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c9f8b namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: fa721402-edb8-44ed-98ab-7a1e1cbb9286 kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c9f8b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:47Z" name: success-200-isvc-c9f8b.18a6debd38e0b189 namespace: kserve-ci-e2e-test resourceVersion: "15334" uid: 9b6a3258-be32-48fd-b65e-dc335874ee08 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c9f8b namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: fa721402-edb8-44ed-98ab-7a1e1cbb9286 kind: Event lastTimestamp: "2026-04-16T15:07:38Z" message: InferenceService [success-200-isvc-c9f8b] is Ready metadata: creationTimestamp: "2026-04-16T15:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:07:38Z" name: success-200-isvc-c9f8b.18a6dec929d39783 namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: c24c5c4a-f806-40f9-a1cd-fc2220fe2285 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:18:12.447144Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595 namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: 22813604-60d6-47d3-baee-bab5f60578ec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-cee76-predictor-84f4bbdc7-bf595 to ip-10-0-130-229.ec2.internal metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:18:12Z" name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595.18a6df5cbd1efe19 namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: 00c9c060-53b2-4f9b-a927-7cb0a91a4ecb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595 namespace: kserve-ci-e2e-test resourceVersion: "21215" uid: 22813604-60d6-47d3-baee-bab5f60578ec kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:18:12Z" name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595.18a6df5cc440153c namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: ccb6aef9-2519-4c3c-a30b-f7343fffd97d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 22813604-60d6-47d3-baee-bab5f60578ec kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1407" already present on machine metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:12Z" name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595.18a6df5cc5675069 namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: 55b699c5-1050-4f54-b9cc-9c4c62a6dfc1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 22813604-60d6-47d3-baee-bab5f60578ec kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:12Z" name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595.18a6df5cc7f062fa namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: cf087690-1f3d-4541-91e1-3194f0a3d0bd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 22813604-60d6-47d3-baee-bab5f60578ec kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:12Z" name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595.18a6df5cc84c303d namespace: kserve-ci-e2e-test resourceVersion: "21260" uid: 517a0c9d-f8b3-446c-a481-7aed85553e3b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:18:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 22813604-60d6-47d3-baee-bab5f60578ec kind: Event lastTimestamp: "2026-04-16T15:18:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:18:44Z" name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595.18a6df5cf4ec6220 namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: cb4289ea-36de-48b3-b5e8-e0fe5c74e8c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 22813604-60d6-47d3-baee-bab5f60578ec kind: Event lastTimestamp: "2026-04-16T15:27:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:27:27Z" name: success-200-isvc-cee76-predictor-84f4bbdc7-bf595.18a6dfddecdb6341 namespace: kserve-ci-e2e-test resourceVersion: "25754" uid: 69e69570-0f5d-485f-a4ea-adea22b24719 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-cee76-predictor-84f4bbdc7 namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: 2816da3c-8ac8-428d-90a8-5c4cd37533ad kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: 'Created pod: success-200-isvc-cee76-predictor-84f4bbdc7-bf595' metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:12Z" name: success-200-isvc-cee76-predictor-84f4bbdc7.18a6df5cbc1f1bf1 namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: 8ef8ae7b-b199-4530-86eb-c106d571bfb2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-cee76-predictor namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: f6bdfc3d-6f7e-4bd1-a7bd-e2ecfdd94d42 kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: Scaled up replica set success-200-isvc-cee76-predictor-84f4bbdc7 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:12Z" name: success-200-isvc-cee76-predictor.18a6df5cb84c34b8 namespace: kserve-ci-e2e-test resourceVersion: "21193" uid: 190e757c-f8d6-46a6-8d62-d06d70f35b0a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cee76-predictor namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: b65b0962-33fb-4bb8-a611-91e2e989f5a8 kind: Event lastTimestamp: "2026-04-16T15:18:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:27Z" name: success-200-isvc-cee76-predictor.18a6df6037b6a369 namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: 396ce7ce-34d5-4de0-8ed9-844e750e0152 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cee76-predictor namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: b65b0962-33fb-4bb8-a611-91e2e989f5a8 kind: Event lastTimestamp: "2026-04-16T15:18:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:18:27Z" name: success-200-isvc-cee76-predictor.18a6df6038124dd3 namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: e669c899-20e5-4fb6-a45d-85a92e5e57cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:18:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cee76-predictor namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: b65b0962-33fb-4bb8-a611-91e2e989f5a8 kind: Event lastTimestamp: "2026-04-16T15:19:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:18:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:19:27Z" name: success-200-isvc-cee76-predictor.18a6df63b70a8e29 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: d6a04dc7-5974-4342-b1d1-37e0204a8ef4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:18:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cee76-predictor namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: b65b0962-33fb-4bb8-a611-91e2e989f5a8 kind: Event lastTimestamp: "2026-04-16T15:19:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:18:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:19:27Z" name: success-200-isvc-cee76-predictor.18a6df63b7623631 namespace: kserve-ci-e2e-test resourceVersion: "21968" uid: 773c7bd6-5c1f-41d0-8432-29d3490d03bf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: e0ee18fa-f97c-4402-bd6e-fadb037d4b39 kind: Event lastTimestamp: "2026-04-16T15:18:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:54Z" name: success-200-isvc-cee76.18a6df5cb8ef4fcd namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: ce5fb59f-e8dc-4f4b-863b-735d410030f4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: e0ee18fa-f97c-4402-bd6e-fadb037d4b39 kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-cee76": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cee76": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:12Z" name: success-200-isvc-cee76.18a6df5cb964f48e namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: a4257459-67dd-48da-a314-5f6d0041eb23 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: e0ee18fa-f97c-4402-bd6e-fadb037d4b39 kind: Event lastTimestamp: "2026-04-16T15:18:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cee76": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:18:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:12Z" name: success-200-isvc-cee76.18a6df5cb9653911 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 1d3cbaca-6d5c-4047-8fd0-a08cc59f4739 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:18:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cee76 namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: e0ee18fa-f97c-4402-bd6e-fadb037d4b39 kind: Event lastTimestamp: "2026-04-16T15:18:54Z" message: InferenceService [success-200-isvc-cee76] is Ready metadata: creationTimestamp: "2026-04-16T15:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:18:54Z" name: success-200-isvc-cee76.18a6df66850b17cd namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: 6e729f3d-aaa1-44a1-b5e4-280c7a466df0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:05:16.948303Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: 735a135c-bde4-4088-bc69-245eb1b4ac4d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp to ip-10-0-130-229.ec2.internal metadata: creationTimestamp: "2026-04-16T15:05:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:05:16Z" name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp.18a6dea82dc9c51e namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: b3f3717a-7980-421b-a7dd-2d7ab1f8740b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: 735a135c-bde4-4088-bc69-245eb1b4ac4d kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:05:17Z" name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp.18a6dea834aa0517 namespace: kserve-ci-e2e-test resourceVersion: "14294" uid: 5410e9d5-fc5b-46df-bdaa-970d71acd457 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 735a135c-bde4-4088-bc69-245eb1b4ac4d kind: Event lastTimestamp: "2026-04-16T15:05:17Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1407" metadata: creationTimestamp: "2026-04-16T15:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:17Z" name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp.18a6dea835d91b92 namespace: kserve-ci-e2e-test resourceVersion: "14297" uid: 429d5852-6bc4-4f8d-8e42-9e4aa867a97b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 735a135c-bde4-4088-bc69-245eb1b4ac4d kind: Event lastTimestamp: "2026-04-16T15:05:30Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1407" in 12.944s (12.944s including waiting). Image size: 1332287104 bytes.' metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:30Z" name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp.18a6deab39618ddb namespace: kserve-ci-e2e-test resourceVersion: "14527" uid: df1e3de3-df16-4320-9425-7fa5056a535d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 735a135c-bde4-4088-bc69-245eb1b4ac4d kind: Event lastTimestamp: "2026-04-16T15:05:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:30Z" name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp.18a6deab3fdd4be3 namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: c99752f7-5927-4153-aa4d-e5dd11423962 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 735a135c-bde4-4088-bc69-245eb1b4ac4d kind: Event lastTimestamp: "2026-04-16T15:05:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:05:30Z" name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp.18a6deab403f9fce namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: b54eb1e7-17f0-43fe-85d4-1d878efea6da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 735a135c-bde4-4088-bc69-245eb1b4ac4d kind: Event lastTimestamp: "2026-04-16T15:06:11Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T15:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:11Z" name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp.18a6deab4b2dd4a3 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: e68c6a8c-3c5c-44b2-904e-230a1c49c8e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 735a135c-bde4-4088-bc69-245eb1b4ac4d kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:47Z" name: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp.18a6debd37804da1 namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 5fb750b7-3237-47eb-83b2-1aa525f1cb2d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-229.ec2.internal source: component: kubelet host: ip-10-0-130-229.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d1f99-predictor-79c6d7db55 namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 49e47a5e-ac65-442b-8571-8c054962d024 kind: Event lastTimestamp: "2026-04-16T15:05:16Z" message: 'Created pod: success-200-isvc-d1f99-predictor-79c6d7db55-qjzfp' metadata: creationTimestamp: "2026-04-16T15:05:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:16Z" name: success-200-isvc-d1f99-predictor-79c6d7db55.18a6dea82d694c5d namespace: kserve-ci-e2e-test resourceVersion: "14278" uid: 8388cc78-da8f-4afb-9546-dbd425b1b6ba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d1f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "14265" uid: c6f4a761-6bf7-41e0-8ee5-36bbec772125 kind: Event lastTimestamp: "2026-04-16T15:05:16Z" message: Scaled up replica set success-200-isvc-d1f99-predictor-79c6d7db55 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:05:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:16Z" name: success-200-isvc-d1f99-predictor.18a6dea826c599b8 namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: b98f7dca-0939-4690-a736-6b21f5af26cc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "14274" uid: 72059b92-bbe5-4f48-b26c-26daf05525ea kind: Event lastTimestamp: "2026-04-16T15:05:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:31Z" name: success-200-isvc-d1f99-predictor.18a6deababc91e28 namespace: kserve-ci-e2e-test resourceVersion: "14580" uid: 8c38b323-f31f-4dd2-9899-d23831f14b15 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:05:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "14274" uid: 72059b92-bbe5-4f48-b26c-26daf05525ea kind: Event lastTimestamp: "2026-04-16T15:05:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:05:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:05:31Z" name: success-200-isvc-d1f99-predictor.18a6deabac24b637 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: c17b5d58-a4ba-432a-9ee4-9b539cb29030 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:05:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "14581" uid: 72059b92-bbe5-4f48-b26c-26daf05525ea kind: Event lastTimestamp: "2026-04-16T15:06:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:05:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:32Z" name: success-200-isvc-d1f99-predictor.18a6deaf2b37a883 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 04ff2b74-09fb-471e-8ef1-c1a40dad5776 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T15:05:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "14581" uid: 72059b92-bbe5-4f48-b26c-26daf05525ea kind: Event lastTimestamp: "2026-04-16T15:06:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:05:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:32Z" name: success-200-isvc-d1f99-predictor.18a6deaf2b906daa namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 17385ef1-9618-4781-b2d8-c0da3a066452 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:05:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1f99 namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: ba837f0a-c552-485b-b574-4db5dcec04af kind: Event lastTimestamp: "2026-04-16T15:06:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T15:05:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:21Z" name: success-200-isvc-d1f99.18a6dea82cdec064 namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: 12e2ebad-fa71-47f6-a5d5-a23beac07d22 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1f99 namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: ba837f0a-c552-485b-b574-4db5dcec04af kind: Event lastTimestamp: "2026-04-16T15:06:21Z" message: InferenceService [success-200-isvc-d1f99] is Ready metadata: creationTimestamp: "2026-04-16T15:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:21Z" name: success-200-isvc-d1f99.18a6deb72ea323bd namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: 836012bc-90b5-4f8c-9a86-a8869501fec8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:28:37.408118Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9c7d6-58455857bf-xj6x5 namespace: kserve-ci-e2e-test resourceVersion: "26609" uid: c6042503-703b-439f-b2a6-fd543b8708e9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-9c7d6-58455857bf-xj6x5 to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:28:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:28:37Z" name: switch-graph-9c7d6-58455857bf-xj6x5.18a6dfee3fb2ab6c namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: cb59f8ee-72f3-4789-829e-f4a35e1eab85 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:37Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9c7d6-58455857bf-xj6x5 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: c6042503-703b-439f-b2a6-fd543b8708e9 kind: Event lastTimestamp: "2026-04-16T15:28:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-9c7d6-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T15:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:28:37Z" name: switch-graph-9c7d6-58455857bf-xj6x5.18a6dfee48fd7bd5 namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: 15377345-d3a0-49a2-9d2e-fd53a6890d04 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:38Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9c7d6-58455857bf-xj6x5 namespace: kserve-ci-e2e-test resourceVersion: "26618" uid: c6042503-703b-439f-b2a6-fd543b8708e9 kind: Event lastTimestamp: "2026-04-16T15:28:38Z" message: Add eth0 [10.134.0.54/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:28:38Z" name: switch-graph-9c7d6-58455857bf-xj6x5.18a6dfee7c86cc2d namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: 24e81e09-8cd5-4db4-86aa-2e3e18e40f20 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c7d6} kind: Pod name: switch-graph-9c7d6-58455857bf-xj6x5 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: c6042503-703b-439f-b2a6-fd543b8708e9 kind: Event lastTimestamp: "2026-04-16T15:28:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" already present on machine metadata: creationTimestamp: "2026-04-16T15:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:28:38Z" name: switch-graph-9c7d6-58455857bf-xj6x5.18a6dfee7db586d3 namespace: kserve-ci-e2e-test resourceVersion: "26637" uid: 34f59e7b-b7ca-4b8b-8038-e8c62eec14f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c7d6} kind: Pod name: switch-graph-9c7d6-58455857bf-xj6x5 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: c6042503-703b-439f-b2a6-fd543b8708e9 kind: Event lastTimestamp: "2026-04-16T15:28:38Z" message: 'Created container: switch-graph-9c7d6' metadata: creationTimestamp: "2026-04-16T15:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:28:38Z" name: switch-graph-9c7d6-58455857bf-xj6x5.18a6dfee80a9fea7 namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: 6f50ecbc-6019-4a53-95c9-038eaeaec9e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c7d6} kind: Pod name: switch-graph-9c7d6-58455857bf-xj6x5 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: c6042503-703b-439f-b2a6-fd543b8708e9 kind: Event lastTimestamp: "2026-04-16T15:28:38Z" message: Started container switch-graph-9c7d6 metadata: creationTimestamp: "2026-04-16T15:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:28:38Z" name: switch-graph-9c7d6-58455857bf-xj6x5.18a6dfee811b2bc4 namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 562ab141-99c1-4944-a903-68f4f19d5725 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:44:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c7d6} kind: Pod name: switch-graph-9c7d6-58455857bf-xj6x5 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: c6042503-703b-439f-b2a6-fd543b8708e9 kind: Event lastTimestamp: "2026-04-16T15:44:56Z" message: Stopping container switch-graph-9c7d6 metadata: creationTimestamp: "2026-04-16T15:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:44:56Z" name: switch-graph-9c7d6-58455857bf-xj6x5.18a6e0d246d6ced1 namespace: kserve-ci-e2e-test resourceVersion: "33470" uid: 88528c80-87b2-4f81-934a-a1742b007326 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T15:45:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c7d6} kind: Pod name: switch-graph-9c7d6-58455857bf-xj6x5 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: c6042503-703b-439f-b2a6-fd543b8708e9 kind: Event lastTimestamp: "2026-04-16T15:45:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:45:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:45:20Z" name: switch-graph-9c7d6-58455857bf-xj6x5.18a6e0d310e1e219 namespace: kserve-ci-e2e-test resourceVersion: "33710" uid: ed806c0b-6c5b-40b6-a698-4f1bfcf0e020 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-9c7d6-58455857bf namespace: kserve-ci-e2e-test resourceVersion: "26606" uid: 5a9f0507-a201-49d9-969a-ffa7d9a426db kind: Event lastTimestamp: "2026-04-16T15:28:37Z" message: 'Created pod: switch-graph-9c7d6-58455857bf-xj6x5' metadata: creationTimestamp: "2026-04-16T15:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:37Z" name: switch-graph-9c7d6-58455857bf.18a6dfee3f50a47c namespace: kserve-ci-e2e-test resourceVersion: "26610" uid: dae3655d-3f45-49ab-be87-b2a0ac4a7f7a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "26604" uid: b2aced6b-0a19-4385-b4ce-44f0a0517c96 kind: Event lastTimestamp: "2026-04-16T15:28:37Z" message: Scaled up replica set switch-graph-9c7d6-58455857bf from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:37Z" name: switch-graph-9c7d6.18a6dfee3ec0b126 namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: 445ec91c-7b52-4193-bb74-87ffb026961c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "26621" uid: 4a0097ba-8b85-4868-96b5-55cd47fe274f kind: Event lastTimestamp: "2026-04-16T15:28:37Z" message: InferenceGraph [switch-graph-9c7d6] is Ready metadata: creationTimestamp: "2026-04-16T15:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:28:37Z" name: switch-graph-9c7d6.18a6dfee422d5b63 namespace: kserve-ci-e2e-test resourceVersion: "26623" uid: 2d9f64e0-2b7a-46d9-b2fd-b96a3aca061a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "26597" uid: 4a0097ba-8b85-4868-96b5-55cd47fe274f kind: Event lastTimestamp: "2026-04-16T15:28:37Z" message: 'Failed to update status for InferenceGraph "switch-graph-9c7d6": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-9c7d6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:28:37Z" name: switch-graph-9c7d6.18a6dfee4427f27d namespace: kserve-ci-e2e-test resourceVersion: "26624" uid: 99505b82-c829-46bd-adf5-445350a64def reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "26597" uid: 4a0097ba-8b85-4868-96b5-55cd47fe274f kind: Event lastTimestamp: "2026-04-16T15:28:37Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-9c7d6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T15:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:28:37Z" name: switch-graph-9c7d6.18a6dfee44283c49 namespace: kserve-ci-e2e-test resourceVersion: "26625" uid: 9fc14edc-ed75-47a5-a7a7-ebb0a8238951 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: df0091e9-2324-4289-9540-17689ed18936 kind: Event lastTimestamp: "2026-04-16T15:28:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:28:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:52Z" name: switch-graph-9c7d6.18a6dff1bd6127fe namespace: kserve-ci-e2e-test resourceVersion: "26799" uid: c497aa2c-649a-41aa-870d-c3ebde39e551 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:28:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: df0091e9-2324-4289-9540-17689ed18936 kind: Event lastTimestamp: "2026-04-16T15:28:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:28:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:28:52Z" name: switch-graph-9c7d6.18a6dff1bdced511 namespace: kserve-ci-e2e-test resourceVersion: "26801" uid: ad29925f-a195-4e71-af3b-95e0cb782485 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:29:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: df0091e9-2324-4289-9540-17689ed18936 kind: Event lastTimestamp: "2026-04-16T15:29:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:29:07Z" name: switch-graph-9c7d6.18a6dff53cb5f397 namespace: kserve-ci-e2e-test resourceVersion: "26915" uid: 870aee0e-59fc-41c7-a9f2-cbb0de7e9cb0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:29:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9c7d6 namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: df0091e9-2324-4289-9540-17689ed18936 kind: Event lastTimestamp: "2026-04-16T15:29:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:29:07Z" name: switch-graph-9c7d6.18a6dff53d30b979 namespace: kserve-ci-e2e-test resourceVersion: "26918" uid: aad9494d-aaf5-4248-99c7-f48a662e0094 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:07:57.442936Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c9f8b-5c958b9899-79ld4 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: d6eed1c0-c2e6-44d1-a359-2a5495a3fb53 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-c9f8b-5c958b9899-79ld4 to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:07:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:07:57Z" name: switch-graph-c9f8b-5c958b9899-79ld4.18a6decd8c037fa6 namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: 88ab5bd1-0c10-404a-a84e-1cda8a0cd130 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:57Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c9f8b-5c958b9899-79ld4 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: d6eed1c0-c2e6-44d1-a359-2a5495a3fb53 kind: Event lastTimestamp: "2026-04-16T15:07:57Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:07:57Z" name: switch-graph-c9f8b-5c958b9899-79ld4.18a6decda593dad9 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: d852831d-5668-467d-a423-127c1e6aa2e9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c9f8b} kind: Pod name: switch-graph-c9f8b-5c958b9899-79ld4 namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: d6eed1c0-c2e6-44d1-a359-2a5495a3fb53 kind: Event lastTimestamp: "2026-04-16T15:07:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" already present on machine metadata: creationTimestamp: "2026-04-16T15:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:57Z" name: switch-graph-c9f8b-5c958b9899-79ld4.18a6decda6b50e6a namespace: kserve-ci-e2e-test resourceVersion: "16231" uid: 3cc1c5ac-8dc3-42b9-b9ef-b578f8a4f7b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c9f8b} kind: Pod name: switch-graph-c9f8b-5c958b9899-79ld4 namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: d6eed1c0-c2e6-44d1-a359-2a5495a3fb53 kind: Event lastTimestamp: "2026-04-16T15:07:57Z" message: 'Created container: switch-graph-c9f8b' metadata: creationTimestamp: "2026-04-16T15:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:57Z" name: switch-graph-c9f8b-5c958b9899-79ld4.18a6decda951c449 namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: f43dc0bd-67b6-4bc3-adda-e78edd58232c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c9f8b} kind: Pod name: switch-graph-c9f8b-5c958b9899-79ld4 namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: d6eed1c0-c2e6-44d1-a359-2a5495a3fb53 kind: Event lastTimestamp: "2026-04-16T15:07:57Z" message: Started container switch-graph-c9f8b metadata: creationTimestamp: "2026-04-16T15:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:57Z" name: switch-graph-c9f8b-5c958b9899-79ld4.18a6decda9be2df1 namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 33790ef9-f85a-443a-b540-98c574234693 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:16:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c9f8b} kind: Pod name: switch-graph-c9f8b-5c958b9899-79ld4 namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: d6eed1c0-c2e6-44d1-a359-2a5495a3fb53 kind: Event lastTimestamp: "2026-04-16T15:16:12Z" message: Stopping container switch-graph-c9f8b metadata: creationTimestamp: "2026-04-16T15:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:12Z" name: switch-graph-c9f8b-5c958b9899-79ld4.18a6df40baee2d8f namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 6aca7e2d-2192-48a9-8272-2dbad4e57ed3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:16:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c9f8b} kind: Pod name: switch-graph-c9f8b-5c958b9899-79ld4 namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: d6eed1c0-c2e6-44d1-a359-2a5495a3fb53 kind: Event lastTimestamp: "2026-04-16T15:16:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:16:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:16:39Z" name: switch-graph-c9f8b-5c958b9899-79ld4.18a6df415be09ba7 namespace: kserve-ci-e2e-test resourceVersion: "20038" uid: a8fc801d-5a38-4c23-9769-fd098aaf9b31 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-c9f8b-5c958b9899 namespace: kserve-ci-e2e-test resourceVersion: "16205" uid: cf6f7029-0f83-4a0b-a985-17341aa3c9ef kind: Event lastTimestamp: "2026-04-16T15:07:57Z" message: 'Created pod: switch-graph-c9f8b-5c958b9899-79ld4' metadata: creationTimestamp: "2026-04-16T15:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:57Z" name: switch-graph-c9f8b-5c958b9899.18a6decd8b9d5ff1 namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: aba6c7cb-2225-4fd4-9a8b-5e4e412b0e89 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-c9f8b namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: 58f2cc65-76fb-4012-8a0d-086a951fa1d8 kind: Event lastTimestamp: "2026-04-16T15:07:57Z" message: Scaled up replica set switch-graph-c9f8b-5c958b9899 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:07:57Z" name: switch-graph-c9f8b.18a6decd8ae71656 namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: a2c32536-faba-4965-85de-e6a6e0bc2d35 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:07:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-c9f8b namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: 3829faea-f0c4-4afd-bb45-5d4b0bc6d8cd kind: Event lastTimestamp: "2026-04-16T15:07:57Z" message: InferenceGraph [switch-graph-c9f8b] is Ready metadata: creationTimestamp: "2026-04-16T15:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:07:57Z" name: switch-graph-c9f8b.18a6decd8e20cdbc namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: 9cecd588-1866-46c8-8bf3-d712c6a7d7c2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c9f8b namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: 39b7d09b-6b8c-4ae3-b243-d06e919650c3 kind: Event lastTimestamp: "2026-04-16T15:08:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:12Z" name: switch-graph-c9f8b.18a6ded10958946b namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: dcf3f144-1c50-4074-971b-ee85544071d2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:08:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c9f8b namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: 39b7d09b-6b8c-4ae3-b243-d06e919650c3 kind: Event lastTimestamp: "2026-04-16T15:08:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T15:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:12Z" name: switch-graph-c9f8b.18a6ded10995e3ce namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 80d7a933-345c-4a6d-b2d6-695b52cd3b7d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:08:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c9f8b namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: 39b7d09b-6b8c-4ae3-b243-d06e919650c3 kind: Event lastTimestamp: "2026-04-16T15:08:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:42Z" name: switch-graph-c9f8b.18a6ded488316f30 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 4bcb8007-053f-42b8-ac14-d38c133ed0b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T15:08:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c9f8b namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: 39b7d09b-6b8c-4ae3-b243-d06e919650c3 kind: Event lastTimestamp: "2026-04-16T15:08:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T15:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:08:42Z" name: switch-graph-c9f8b.18a6ded4888f83d4 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 674f35fa-7f13-41d4-ac46-d7aa3392b177 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T15:06:36.970704Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d1f99-5f9847b898-pp994 namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 8f2203ad-4be6-4d90-a337-521909c216ac kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d1f99-5f9847b898-pp994 to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-04-16T15:06:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T15:06:36Z" name: switch-graph-d1f99-5f9847b898-pp994.18a6debacf7eb056 namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: df0db304-976b-43c5-9820-298fcf3a9dcc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df5c5877c-k6v7x source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:37Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d1f99-5f9847b898-pp994 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: 8f2203ad-4be6-4d90-a337-521909c216ac kind: Event lastTimestamp: "2026-04-16T15:06:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-d1f99-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T15:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:37Z" name: switch-graph-d1f99-5f9847b898-pp994.18a6debadbafc63a namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 18847fba-341f-42da-9993-50e45cf0460d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:37Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d1f99-5f9847b898-pp994 namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: 8f2203ad-4be6-4d90-a337-521909c216ac kind: Event lastTimestamp: "2026-04-16T15:06:37Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T15:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T15:06:37Z" name: switch-graph-d1f99-5f9847b898-pp994.18a6debb0c4dc94c namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: f065a0ba-7744-415a-afed-e437c5586569 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d1f99} kind: Pod name: switch-graph-d1f99-5f9847b898-pp994 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: 8f2203ad-4be6-4d90-a337-521909c216ac kind: Event lastTimestamp: "2026-04-16T15:06:38Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" metadata: creationTimestamp: "2026-04-16T15:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:38Z" name: switch-graph-d1f99-5f9847b898-pp994.18a6debb0d82d1ca namespace: kserve-ci-e2e-test resourceVersion: "15192" uid: 9d7a6103-4470-4643-a883-45cf2e2854d6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d1f99} kind: Pod name: switch-graph-d1f99-5f9847b898-pp994 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: 8f2203ad-4be6-4d90-a337-521909c216ac kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:daaaf0b7769b899c00520a668e190161eefaa93b6eb75bfb8161ae18d89812c6" in 1.446s (1.446s including waiting). Image size: 216232057 bytes.' metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:39Z" name: switch-graph-d1f99-5f9847b898-pp994.18a6debb63bad76c namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: 19903dde-b016-45cd-b180-1fb260dd1dac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d1f99} kind: Pod name: switch-graph-d1f99-5f9847b898-pp994 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: 8f2203ad-4be6-4d90-a337-521909c216ac kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: 'Created container: switch-graph-d1f99' metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:39Z" name: switch-graph-d1f99-5f9847b898-pp994.18a6debb67053e4c namespace: kserve-ci-e2e-test resourceVersion: "15206" uid: ad31ad50-c4d1-487d-9f6c-07be177b449e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d1f99} kind: Pod name: switch-graph-d1f99-5f9847b898-pp994 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: 8f2203ad-4be6-4d90-a337-521909c216ac kind: Event lastTimestamp: "2026-04-16T15:06:39Z" message: Started container switch-graph-d1f99 metadata: creationTimestamp: "2026-04-16T15:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:39Z" name: switch-graph-d1f99-5f9847b898-pp994.18a6debb6777fc2e namespace: kserve-ci-e2e-test resourceVersion: "15207" uid: 07d00985-cd45-4e69-a76d-e5e0e3b4d311 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d1f99} kind: Pod name: switch-graph-d1f99-5f9847b898-pp994 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: 8f2203ad-4be6-4d90-a337-521909c216ac kind: Event lastTimestamp: "2026-04-16T15:06:47Z" message: Stopping container switch-graph-d1f99 metadata: creationTimestamp: "2026-04-16T15:06:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:06:47Z" name: switch-graph-d1f99-5f9847b898-pp994.18a6debd3199419d namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: aeba3d42-b60a-4830-a2e9-3a83fff0b368 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T15:06:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d1f99} kind: Pod name: switch-graph-d1f99-5f9847b898-pp994 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: 8f2203ad-4be6-4d90-a337-521909c216ac kind: Event lastTimestamp: "2026-04-16T15:07:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T15:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T15:07:15Z" name: switch-graph-d1f99-5f9847b898-pp994.18a6debdf9a4716c namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: dae41bfc-9200-42b6-810c-06abc719d166 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d1f99-5f9847b898 namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: 012fbb9d-3ac5-41c0-83a3-ed9acf65d5eb kind: Event lastTimestamp: "2026-04-16T15:06:36Z" message: 'Created pod: switch-graph-d1f99-5f9847b898-pp994' metadata: creationTimestamp: "2026-04-16T15:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:36Z" name: switch-graph-d1f99-5f9847b898.18a6debacf1432b7 namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: 673215f5-d078-49ee-a0d5-2894296789e9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d1f99 namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 7e624418-81d0-4d1a-a753-1cad316219d4 kind: Event lastTimestamp: "2026-04-16T15:06:36Z" message: Scaled up replica set switch-graph-d1f99-5f9847b898 from 0 to 1 metadata: creationTimestamp: "2026-04-16T15:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T15:06:36Z" name: switch-graph-d1f99.18a6debace7109d1 namespace: kserve-ci-e2e-test resourceVersion: "15161" uid: 061d47f2-0dc3-4d84-9be2-07e3bd67cc18 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T15:06:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d1f99 namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: 032548dd-6c27-4ee3-ba50-816aa431d38d kind: Event lastTimestamp: "2026-04-16T15:06:37Z" message: InferenceGraph [switch-graph-d1f99] is Ready metadata: creationTimestamp: "2026-04-16T15:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T15:06:37Z" name: switch-graph-d1f99.18a6debad2e006e7 namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: 38b15652-4123-4191-a1cc-43ad25735f71 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "33714"