--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:08:12.793976Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-01c7a-7f54567bd4-96nwd namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: 65eab692-528b-4c72-bc0b-4fdb2f5b4678 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-01c7a-7f54567bd4-96nwd to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:08:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:08:12Z" name: ensemble-graph-01c7a-7f54567bd4-96nwd.18c785cd00358646 namespace: kserve-ci-e2e-test resourceVersion: "21535" uid: 3d046938-880e-4f05-9682-65b16b7a8989 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:12Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-01c7a-7f54567bd4-96nwd namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: 65eab692-528b-4c72-bc0b-4fdb2f5b4678 kind: Event lastTimestamp: "2026-08-01T00:08:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-01c7a-serving-cert" not found' metadata: creationTimestamp: "2026-08-01T00:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:12Z" name: ensemble-graph-01c7a-7f54567bd4-96nwd.18c785cd0723d797 namespace: kserve-ci-e2e-test resourceVersion: "21548" uid: 4e93754f-9c26-4856-b44a-5a30e18edd4d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:13Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-01c7a-7f54567bd4-96nwd namespace: kserve-ci-e2e-test resourceVersion: "21541" uid: 65eab692-528b-4c72-bc0b-4fdb2f5b4678 kind: Event lastTimestamp: "2026-08-01T00:08:13Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:08:13Z" name: ensemble-graph-01c7a-7f54567bd4-96nwd.18c785cd3deb1205 namespace: kserve-ci-e2e-test resourceVersion: "21556" uid: b0e71205-f607-4a6d-a5cd-87a302483475 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-01c7a} kind: Pod name: ensemble-graph-01c7a-7f54567bd4-96nwd namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: 65eab692-528b-4c72-bc0b-4fdb2f5b4678 kind: Event lastTimestamp: "2026-08-01T00:08:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:56dbaa30789264fbe80634bdc08e7f43bcfc19e5006180ec98cfd4cb6b8d99d6" already present on machine metadata: creationTimestamp: "2026-08-01T00:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:13Z" name: ensemble-graph-01c7a-7f54567bd4-96nwd.18c785cd3f30d0b9 namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: f0b8125b-8b5c-4d31-b0ce-533600a6821f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-01c7a} kind: Pod name: ensemble-graph-01c7a-7f54567bd4-96nwd namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: 65eab692-528b-4c72-bc0b-4fdb2f5b4678 kind: Event lastTimestamp: "2026-08-01T00:08:13Z" message: 'Created container: ensemble-graph-01c7a' metadata: creationTimestamp: "2026-08-01T00:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:13Z" name: ensemble-graph-01c7a-7f54567bd4-96nwd.18c785cd424000b5 namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: 7df8d1a3-cb53-4418-bb27-4442802e6e11 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-01c7a} kind: Pod name: ensemble-graph-01c7a-7f54567bd4-96nwd namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: 65eab692-528b-4c72-bc0b-4fdb2f5b4678 kind: Event lastTimestamp: "2026-08-01T00:08:13Z" message: Started container ensemble-graph-01c7a metadata: creationTimestamp: "2026-08-01T00:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:13Z" name: ensemble-graph-01c7a-7f54567bd4-96nwd.18c785cd42cdae06 namespace: kserve-ci-e2e-test resourceVersion: "21561" uid: df4987ed-1fb8-4c73-8b94-91b94c1d27fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-01c7a} kind: Pod name: ensemble-graph-01c7a-7f54567bd4-96nwd namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: 65eab692-528b-4c72-bc0b-4fdb2f5b4678 kind: Event lastTimestamp: "2026-08-01T00:16:27Z" message: Stopping container ensemble-graph-01c7a metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:27Z" name: ensemble-graph-01c7a-7f54567bd4-96nwd.18c786402e85c2c3 namespace: kserve-ci-e2e-test resourceVersion: "25041" uid: 2ad10f75-0d2d-45e8-bf32-ff586e53813c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:16:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-01c7a} kind: Pod name: ensemble-graph-01c7a-7f54567bd4-96nwd namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: 65eab692-528b-4c72-bc0b-4fdb2f5b4678 kind: Event lastTimestamp: "2026-08-01T00:16:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-01T00:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:55Z" name: ensemble-graph-01c7a-7f54567bd4-96nwd.18c78640ec1f73e0 namespace: kserve-ci-e2e-test resourceVersion: "25654" uid: d085a4ef-6858-4bd2-ac30-75b4d0330cf8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-01c7a-7f54567bd4 namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: 78fbc1bd-58e6-461c-92b8-dc6ad483e2e3 kind: Event lastTimestamp: "2026-08-01T00:08:12Z" message: 'Created pod: ensemble-graph-01c7a-7f54567bd4-96nwd' metadata: creationTimestamp: "2026-08-01T00:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:12Z" name: ensemble-graph-01c7a-7f54567bd4.18c785ccff5b75c0 namespace: kserve-ci-e2e-test resourceVersion: "21532" uid: 3954265a-6d99-482c-9d02-489e82913714 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-01c7a namespace: kserve-ci-e2e-test resourceVersion: "21523" uid: 1f9200fb-5c4e-4bbd-bfcf-a1aa10ece428 kind: Event lastTimestamp: "2026-08-01T00:08:12Z" message: Scaled up replica set ensemble-graph-01c7a-7f54567bd4 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:12Z" name: ensemble-graph-01c7a.18c785ccfd88cfc8 namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: ff5e1eca-1fb5-41b0-86ea-b5476d698e4c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-01c7a namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: 8ed62c46-750e-419e-8bfc-2f23714116cc kind: Event lastTimestamp: "2026-08-01T00:08:12Z" message: InferenceGraph [ensemble-graph-01c7a] is Ready metadata: creationTimestamp: "2026-08-01T00:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:08:12Z" name: ensemble-graph-01c7a.18c785cd03d4e9bc namespace: kserve-ci-e2e-test resourceVersion: "21546" uid: 5ad96534-b1c2-4679-be53-f2b978bfaeaf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:08:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-01c7a namespace: kserve-ci-e2e-test resourceVersion: "21525" uid: 451b44e2-29e8-488f-a37c-474739a3c569 kind: Event lastTimestamp: "2026-08-01T00:08:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:42Z" name: ensemble-graph-01c7a.18c785d07b7dafbf namespace: kserve-ci-e2e-test resourceVersion: "21796" uid: 082b7cd0-5e98-4c57-81f6-22db0ea7530c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:08:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-01c7a namespace: kserve-ci-e2e-test resourceVersion: "21525" uid: 451b44e2-29e8-488f-a37c-474739a3c569 kind: Event lastTimestamp: "2026-08-01T00:08:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:42Z" name: ensemble-graph-01c7a.18c785d07be12043 namespace: kserve-ci-e2e-test resourceVersion: "21798" uid: 02e8240c-bc38-47fa-bbe4-f579ad02643d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:07:02.363472Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-fe2ca-7b566d4bb8-77nlb namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: 00818bdd-5ee9-4a26-ba26-0ecadbf181c8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-fe2ca-7b566d4bb8-77nlb to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:07:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:07:02Z" name: ensemble-graph-fe2ca-7b566d4bb8-77nlb.18c785bc9a3944ab namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: d0b50299-e2d9-4c57-be1d-18ec7459c278 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:02Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-fe2ca-7b566d4bb8-77nlb namespace: kserve-ci-e2e-test resourceVersion: "20566" uid: 00818bdd-5ee9-4a26-ba26-0ecadbf181c8 kind: Event lastTimestamp: "2026-08-01T00:07:02Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:07:02Z" name: ensemble-graph-fe2ca-7b566d4bb8-77nlb.18c785bcb45267b1 namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: 9494712b-64d3-4e75-bbb7-1c68202fd0bf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fe2ca} kind: Pod name: ensemble-graph-fe2ca-7b566d4bb8-77nlb namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 00818bdd-5ee9-4a26-ba26-0ecadbf181c8 kind: Event lastTimestamp: "2026-08-01T00:07:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:56dbaa30789264fbe80634bdc08e7f43bcfc19e5006180ec98cfd4cb6b8d99d6" already present on machine metadata: creationTimestamp: "2026-08-01T00:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:02Z" name: ensemble-graph-fe2ca-7b566d4bb8-77nlb.18c785bcb5bfe81d namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: 4987d02e-7608-4787-ae6b-8c201e408ef3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fe2ca} kind: Pod name: ensemble-graph-fe2ca-7b566d4bb8-77nlb namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 00818bdd-5ee9-4a26-ba26-0ecadbf181c8 kind: Event lastTimestamp: "2026-08-01T00:07:02Z" message: 'Created container: ensemble-graph-fe2ca' metadata: creationTimestamp: "2026-08-01T00:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:02Z" name: ensemble-graph-fe2ca-7b566d4bb8-77nlb.18c785bcb8b81c80 namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: 1cdfb603-0d1a-413e-baa2-e346a87e27f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fe2ca} kind: Pod name: ensemble-graph-fe2ca-7b566d4bb8-77nlb namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 00818bdd-5ee9-4a26-ba26-0ecadbf181c8 kind: Event lastTimestamp: "2026-08-01T00:07:02Z" message: Started container ensemble-graph-fe2ca metadata: creationTimestamp: "2026-08-01T00:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:02Z" name: ensemble-graph-fe2ca-7b566d4bb8-77nlb.18c785bcb92e7408 namespace: kserve-ci-e2e-test resourceVersion: "20583" uid: 7ceb866c-50cb-4821-93e5-66953303619f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fe2ca} kind: Pod name: ensemble-graph-fe2ca-7b566d4bb8-77nlb namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 00818bdd-5ee9-4a26-ba26-0ecadbf181c8 kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: Stopping container ensemble-graph-fe2ca metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:12Z" name: ensemble-graph-fe2ca-7b566d4bb8-77nlb.18c785bef42a44b6 namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: 34afa1d8-e7ba-49d7-a6a9-69e9cc84183c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:07:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fe2ca} kind: Pod name: ensemble-graph-fe2ca-7b566d4bb8-77nlb namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 00818bdd-5ee9-4a26-ba26-0ecadbf181c8 kind: Event lastTimestamp: "2026-08-01T00:07:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-01T00:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:39Z" name: ensemble-graph-fe2ca-7b566d4bb8-77nlb.18c785bf61dcea02 namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: 929d36b6-27f7-47e3-8b0c-d7da4d96a96a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-fe2ca-7b566d4bb8 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: bdf189fa-6d8a-4fbc-a9da-805b3878fe44 kind: Event lastTimestamp: "2026-08-01T00:07:02Z" message: 'Created pod: ensemble-graph-fe2ca-7b566d4bb8-77nlb' metadata: creationTimestamp: "2026-08-01T00:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:02Z" name: ensemble-graph-fe2ca-7b566d4bb8.18c785bc99d4a3d2 namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: 4b7262d5-c33f-4637-b012-3b4cd9ec9bc0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-fe2ca namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 78263466-9f70-4485-a3b4-a6d49ff1e030 kind: Event lastTimestamp: "2026-08-01T00:07:02Z" message: Scaled up replica set ensemble-graph-fe2ca-7b566d4bb8 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:02Z" name: ensemble-graph-fe2ca.18c785bc992d1fb1 namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 73ac2380-1760-4f40-aa4e-ba7965694513 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-fe2ca namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: 06eb0b4a-ec43-4561-88b5-70f5793ca7d3 kind: Event lastTimestamp: "2026-08-01T00:07:02Z" message: InferenceGraph [ensemble-graph-fe2ca] is Ready metadata: creationTimestamp: "2026-08-01T00:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:07:02Z" name: ensemble-graph-fe2ca.18c785bc9ea7654c namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: a63c7944-7c58-49bc-a127-68c89555f529 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:07:12.769189Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 50116272-1adb-4354-9026-e97947524575 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-01c7a-predictor-86b4c76644-xbdpz to ip-10-0-128-75.ec2.internal metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:07:12Z" name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz.18c785bf0673e294 namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: f978b69b-adea-4ebb-9eec-a68b2909f328 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 50116272-1adb-4354-9026-e97947524575 kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-01c7a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:12Z" name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz.18c785bf13624796 namespace: kserve-ci-e2e-test resourceVersion: "20780" uid: 99c86fb9-bb42-4cfd-a95f-206d69915743 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz namespace: kserve-ci-e2e-test resourceVersion: "20771" uid: 50116272-1adb-4354-9026-e97947524575 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:07:13Z" name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz.18c785bf443a555a namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: 7b3850d8-b5ab-4c54-a271-f8b18ccb4b9e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 50116272-1adb-4354-9026-e97947524575 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:13Z" name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz.18c785bf458ecb9d namespace: kserve-ci-e2e-test resourceVersion: "20798" uid: 5391e95e-15a3-4340-b304-bb948321452d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 50116272-1adb-4354-9026-e97947524575 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:13Z" name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz.18c785bf4885635f namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: a75ae511-4fde-4582-9c7d-54094131d820 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 50116272-1adb-4354-9026-e97947524575 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:13Z" name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz.18c785bf48febb34 namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: e2b10aad-e613-41c8-b29d-40f073d5e23b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 50116272-1adb-4354-9026-e97947524575 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:13Z" name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz.18c785bf4909c755 namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: 813d7c06-7fe0-44dd-abf2-eb68b37ecbab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 50116272-1adb-4354-9026-e97947524575 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:13Z" name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz.18c785bf4c737d8c namespace: kserve-ci-e2e-test resourceVersion: "20806" uid: f7de39ce-4b53-4d9f-a06a-67808979b089 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 50116272-1adb-4354-9026-e97947524575 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:13Z" name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz.18c785bf4d15aee1 namespace: kserve-ci-e2e-test resourceVersion: "20807" uid: e79a40f0-e7d3-4c8f-bb3b-cc674cf7474e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:07:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 50116272-1adb-4354-9026-e97947524575 kind: Event lastTimestamp: "2026-08-01T00:07:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:51Z" name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz.18c785bf9f5ca6ad namespace: kserve-ci-e2e-test resourceVersion: "21150" uid: 00df8cb4-e0dc-463d-9038-05201233a605 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 50116272-1adb-4354-9026-e97947524575 kind: Event lastTimestamp: "2026-08-01T00:16:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:27Z" name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz.18c786403d513380 namespace: kserve-ci-e2e-test resourceVersion: "25116" uid: 043617dd-0d4e-4d22-b0cb-7f67d3c210c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 50116272-1adb-4354-9026-e97947524575 kind: Event lastTimestamp: "2026-08-01T00:16:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:27Z" name: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz.18c786403d51e402 namespace: kserve-ci-e2e-test resourceVersion: "25118" uid: 62ed6624-c0cd-479f-bdb6-5874db75d9b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-01c7a-predictor-86b4c76644 namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 420f9fc2-db52-47bb-8c84-7b23dee65ef1 kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: 'Created pod: error-404-isvc-01c7a-predictor-86b4c76644-xbdpz' metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:12Z" name: error-404-isvc-01c7a-predictor-86b4c76644.18c785bf060d3e53 namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: 84bf0269-20f5-4988-948b-4e3aa14ee6c2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-01c7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20750" uid: 40ca0e0f-85e0-4273-8622-247a065f9fb9 kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: Scaled up replica set error-404-isvc-01c7a-predictor-86b4c76644 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:12Z" name: error-404-isvc-01c7a-predictor.18c785bf042eb6a2 namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 6b7f0d7c-d3f1-41ae-ad4c-1a99d4f374c3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:07:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-01c7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: 2875fbf8-c1cc-464b-8283-2087d9ff929a kind: Event lastTimestamp: "2026-08-01T00:07:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:42Z" name: error-404-isvc-01c7a-predictor.18c785c28295316a namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: 68c28d2a-165e-4972-9411-7ae84f3f6063 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:07:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-01c7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: 2875fbf8-c1cc-464b-8283-2087d9ff929a kind: Event lastTimestamp: "2026-08-01T00:07:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:42Z" name: error-404-isvc-01c7a-predictor.18c785c282fa3c6d namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: bc9da93a-248d-43ce-8566-150c5f158d8c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:07:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-01c7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20944" uid: 2875fbf8-c1cc-464b-8283-2087d9ff929a kind: Event lastTimestamp: "2026-08-01T00:08:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:27Z" name: error-404-isvc-01c7a-predictor.18c785c9805ff0c7 namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: b4ae356e-996b-489b-8a74-093e6ad43f38 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:07:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-01c7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20944" uid: 2875fbf8-c1cc-464b-8283-2087d9ff929a kind: Event lastTimestamp: "2026-08-01T00:08:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:27Z" name: error-404-isvc-01c7a-predictor.18c785c980c33ffa namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: 3128266c-981b-485a-bfc7-c504328e698e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-01c7a namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: 995030d7-2316-4dd2-b111-4f44d6d15903 kind: Event lastTimestamp: "2026-08-01T00:08:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:08:01Z" name: error-404-isvc-01c7a.18c785bf045a9b52 namespace: kserve-ci-e2e-test resourceVersion: "21284" uid: 465f3327-5161-4e03-aebf-40d0b163d334 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-01c7a namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: 995030d7-2316-4dd2-b111-4f44d6d15903 kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-01c7a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-01c7a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:07:12Z" name: error-404-isvc-01c7a.18c785bf04dd4c56 namespace: kserve-ci-e2e-test resourceVersion: "20758" uid: cc9fdb3b-1fb7-40b8-b5b3-8f7e42cf6fc3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-01c7a namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: 995030d7-2316-4dd2-b111-4f44d6d15903 kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-01c7a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:07:12Z" name: error-404-isvc-01c7a.18c785bf04dd9d78 namespace: kserve-ci-e2e-test resourceVersion: "20760" uid: 06ac186d-84c4-43c8-a32f-ea9a2f29e8cc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-01c7a namespace: kserve-ci-e2e-test resourceVersion: "21281" uid: 995030d7-2316-4dd2-b111-4f44d6d15903 kind: Event lastTimestamp: "2026-08-01T00:08:01Z" message: InferenceService [error-404-isvc-01c7a] is Ready metadata: creationTimestamp: "2026-08-01T00:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:08:01Z" name: error-404-isvc-01c7a.18c785ca5a40f1cb namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: a5823fd5-0668-45c6-8d08-2db5fbc442d8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:17:38.136403Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52 namespace: kserve-ci-e2e-test resourceVersion: "26296" uid: bbd728e2-bb32-4672-961a-930b5d3c4a6d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2f013-predictor-6456f4cb4-hjp52 to ip-10-0-128-75.ec2.internal metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:17:38Z" name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52.18c78650a13e42f7 namespace: kserve-ci-e2e-test resourceVersion: "26300" uid: c2130e13-57a1-4994-9662-3bee28842af8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52 namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: bbd728e2-bb32-4672-961a-930b5d3c4a6d kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:17:38Z" name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52.18c78650bb489566 namespace: kserve-ci-e2e-test resourceVersion: "26323" uid: a7f040b8-c444-41be-9a91-a08bf6d1e7d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52 namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: bbd728e2-bb32-4672-961a-930b5d3c4a6d kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52.18c78650bc97b062 namespace: kserve-ci-e2e-test resourceVersion: "26326" uid: 58cc2c39-51a5-45f0-b277-01a4fd39822f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52 namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: bbd728e2-bb32-4672-961a-930b5d3c4a6d kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52.18c78650bf60bf2e namespace: kserve-ci-e2e-test resourceVersion: "26329" uid: bc66dcb3-e5ae-4bf5-abf5-a78c001cd9f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52 namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: bbd728e2-bb32-4672-961a-930b5d3c4a6d kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52.18c78650bfdd8d09 namespace: kserve-ci-e2e-test resourceVersion: "26330" uid: e87d3657-ae2a-4423-beab-38dc0483f539 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52 namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: bbd728e2-bb32-4672-961a-930b5d3c4a6d kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52.18c78650bfe7a37f namespace: kserve-ci-e2e-test resourceVersion: "26331" uid: 8b5568f5-b725-470b-81e6-6d457b57a473 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52 namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: bbd728e2-bb32-4672-961a-930b5d3c4a6d kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52.18c78650c421451b namespace: kserve-ci-e2e-test resourceVersion: "26332" uid: d8929d72-a377-4abb-8e3e-ac195ecb753d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52 namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: bbd728e2-bb32-4672-961a-930b5d3c4a6d kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52.18c78650c4bfcc58 namespace: kserve-ci-e2e-test resourceVersion: "26333" uid: bc6bceb0-aedd-488b-9312-227d88f18084 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-01T00:17:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52 namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: bbd728e2-bb32-4672-961a-930b5d3c4a6d kind: Event lastTimestamp: "2026-08-01T00:26:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:17:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:26:55Z" name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52.18c786510160fad3 namespace: kserve-ci-e2e-test resourceVersion: "30429" uid: 30741b1e-7982-459f-bc14-5687bd6e75db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:26:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52 namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: bbd728e2-bb32-4672-961a-930b5d3c4a6d kind: Event lastTimestamp: "2026-08-01T00:26:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:26:52Z" name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52.18c786d1ceec6b62 namespace: kserve-ci-e2e-test resourceVersion: "30404" uid: 2fc35549-6591-4208-aecc-4022d320ea35 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:26:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52 namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: bbd728e2-bb32-4672-961a-930b5d3c4a6d kind: Event lastTimestamp: "2026-08-01T00:26:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:26:52Z" name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52.18c786d1ceedc496 namespace: kserve-ci-e2e-test resourceVersion: "30405" uid: dc1e4334-9a32-412b-aed8-b4a5eaf266ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52 namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: bbd728e2-bb32-4672-961a-930b5d3c4a6d kind: Event lastTimestamp: "2026-08-01T00:26:55Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:26:55Z" name: error-404-isvc-2f013-predictor-6456f4cb4-hjp52.18c786d275bf1381 namespace: kserve-ci-e2e-test resourceVersion: "30428" uid: e30e35a0-179f-4938-ae80-6c99168be7a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2f013-predictor-6456f4cb4 namespace: kserve-ci-e2e-test resourceVersion: "26289" uid: 8a95db9c-cc80-4c90-b5af-e299c28a1472 kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: 'Created pod: error-404-isvc-2f013-predictor-6456f4cb4-hjp52' metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:38Z" name: error-404-isvc-2f013-predictor-6456f4cb4.18c78650a0b0c1d2 namespace: kserve-ci-e2e-test resourceVersion: "26297" uid: a3d565de-feb0-4d17-ad9c-7bef94cea8ea reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2f013-predictor namespace: kserve-ci-e2e-test resourceVersion: "26286" uid: 3a8b9d8c-0dbd-4c09-87f1-da265c83dc61 kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: Scaled up replica set error-404-isvc-2f013-predictor-6456f4cb4 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:38Z" name: error-404-isvc-2f013-predictor.18c786509f13e2fd namespace: kserve-ci-e2e-test resourceVersion: "26291" uid: 5681483a-8769-4c7b-8493-2054ff5301b0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:17:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2f013-predictor namespace: kserve-ci-e2e-test resourceVersion: "26288" uid: 2c73b5a2-bce3-45e4-88e8-3d391ce435a3 kind: Event lastTimestamp: "2026-08-01T00:18:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:08Z" name: error-404-isvc-2f013-predictor.18c786541dc6d814 namespace: kserve-ci-e2e-test resourceVersion: "26624" uid: 422a94ce-e904-401f-a768-b7515d2764f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:17:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2f013-predictor namespace: kserve-ci-e2e-test resourceVersion: "26288" uid: 2c73b5a2-bce3-45e4-88e8-3d391ce435a3 kind: Event lastTimestamp: "2026-08-01T00:18:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:08Z" name: error-404-isvc-2f013-predictor.18c786541e1cb852 namespace: kserve-ci-e2e-test resourceVersion: "26625" uid: 0949f366-3cda-4da7-8ea1-3edd9fd200f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:18:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2f013-predictor namespace: kserve-ci-e2e-test resourceVersion: "26476" uid: 2c73b5a2-bce3-45e4-88e8-3d391ce435a3 kind: Event lastTimestamp: "2026-08-01T00:18:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:18:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:38Z" name: error-404-isvc-2f013-predictor.18c7865b1b7ff210 namespace: kserve-ci-e2e-test resourceVersion: "26940" uid: 8f1db6de-48b2-46ab-b418-8bd628e68655 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:18:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2f013-predictor namespace: kserve-ci-e2e-test resourceVersion: "26476" uid: 2c73b5a2-bce3-45e4-88e8-3d391ce435a3 kind: Event lastTimestamp: "2026-08-01T00:18:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:18:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:38Z" name: error-404-isvc-2f013-predictor.18c7865b1be845d1 namespace: kserve-ci-e2e-test resourceVersion: "26941" uid: 4a034de6-363b-46ed-b655-10379be67c13 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2f013 namespace: kserve-ci-e2e-test resourceVersion: "26272" uid: d755daa1-04d8-45cf-b59d-477554b982c7 kind: Event lastTimestamp: "2026-08-01T00:18:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:18:25Z" name: error-404-isvc-2f013.18c786509f4d9e9f namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: 625a574f-71ab-4f73-b18e-9caa470d9715 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2f013 namespace: kserve-ci-e2e-test resourceVersion: "26807" uid: d755daa1-04d8-45cf-b59d-477554b982c7 kind: Event lastTimestamp: "2026-08-01T00:18:25Z" message: InferenceService [error-404-isvc-2f013] is Ready metadata: creationTimestamp: "2026-08-01T00:18:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:18:25Z" name: error-404-isvc-2f013.18c7865bbc0f89c1 namespace: kserve-ci-e2e-test resourceVersion: "26808" uid: 43dae040-65d6-47a2-a4ac-4bf98fab6049 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T23:56:47.327730Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15450" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 to ip-10-0-128-75.ec2.internal metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T23:56:47Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c7852d673c95fd namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: 9a8533f1-d5b0-4eee-b544-1b306b313398 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: "2026-07-31T23:56:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-32541-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:47Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c7852d732f7854 namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: 9bf657a5-0b20-444d-85ac-01356f7a60c8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T23:56:48Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c7852da4c6cd6a namespace: kserve-ci-e2e-test resourceVersion: "15490" uid: 734db5e8-976f-4ebf-ae80-15cf44087fca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:48Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c7852da611840f namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 0e981ef3-0ded-4e63-aada-67fdac603e57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:48Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c7852da91eb034 namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: bed7bf6e-b8f3-4dba-8914-5205d0f7ce4e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:48Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c7852da9a37d69 namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 5f03b354-5a73-4472-8c20-698fd4695633 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:48Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c7852da9adb135 namespace: kserve-ci-e2e-test resourceVersion: "15497" uid: 31347b9e-d717-45f1-b416-504bd95d3737 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:48Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c7852dacc31a0d namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: fa369141-4e6b-442d-a367-47e217aaaa62 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:48Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c7852dad35b964 namespace: kserve-ci-e2e-test resourceVersion: "15499" uid: 7e700a90-4125-483d-9c07-9e07fdf5d254 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T23:56:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: "2026-07-31T23:57:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:25Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c7852e02c865b4 namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: c96bed42-6e5a-48ff-8cad-33edcea7d0a0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c785aea210dc2c namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 76250b52-b902-49aa-a5af-543e6dbf2b83 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c785aea21161a8 namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 987ea90d-b53e-4d75-87ca-cfeaff09e1e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: "2026-08-01T00:06:06Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-08-01T00:06:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:06Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c785afb2ee7b91 namespace: kserve-ci-e2e-test resourceVersion: "19962" uid: 80a6fe65-fba4-4c66-a45f-90d6217c0dc4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 097e1321-7c6d-4446-98b2-21cc0442df39 kind: Event lastTimestamp: "2026-08-01T00:06:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: i/o timeout' metadata: creationTimestamp: "2026-08-01T00:06:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:06Z" name: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6.18c785afb33d7a40 namespace: kserve-ci-e2e-test resourceVersion: "19963" uid: 128abc82-1818-43ef-bc17-73b43cb84780 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-32541-predictor-8d6ffd9d namespace: kserve-ci-e2e-test resourceVersion: "15442" uid: 9853f68a-281a-4a0d-ad0f-1b2f92063f3c kind: Event lastTimestamp: "2026-07-31T23:56:47Z" message: 'Created pod: error-404-isvc-32541-predictor-8d6ffd9d-mp9l6' metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:47Z" name: error-404-isvc-32541-predictor-8d6ffd9d.18c7852d66b3c467 namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: dd88fcf2-60ad-444e-9348-b63d1238ba3d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-32541-predictor namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: 77919b7b-01d0-4816-b0ac-419dc3187bd2 kind: Event lastTimestamp: "2026-07-31T23:56:47Z" message: Scaled up replica set error-404-isvc-32541-predictor-8d6ffd9d from 0 to 1 metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:47Z" name: error-404-isvc-32541-predictor.18c7852d6521edef namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 4201c54e-c2d7-477f-8b6c-05134b3ab727 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32541-predictor namespace: kserve-ci-e2e-test resourceVersion: "15441" uid: 8e2560a7-be0f-45e9-9ec3-cba214f83c59 kind: Event lastTimestamp: "2026-07-31T23:57:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:02Z" name: error-404-isvc-32541-predictor.18c78530e3a5d0ad namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: 58021d82-0fab-410e-8b8f-4768305c744e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32541-predictor namespace: kserve-ci-e2e-test resourceVersion: "15441" uid: 8e2560a7-be0f-45e9-9ec3-cba214f83c59 kind: Event lastTimestamp: "2026-07-31T23:57:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:02Z" name: error-404-isvc-32541-predictor.18c78530e3da54f9 namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: a94f1f23-0bb9-4eac-9a2b-1a3421b9527a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T23:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32541-predictor namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: 8e2560a7-be0f-45e9-9ec3-cba214f83c59 kind: Event lastTimestamp: "2026-07-31T23:58:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:02Z" name: error-404-isvc-32541-predictor.18c7853462678b2e namespace: kserve-ci-e2e-test resourceVersion: "16407" uid: b4137bb0-bca0-46b1-81aa-7c86ad00f848 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T23:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32541-predictor namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: 8e2560a7-be0f-45e9-9ec3-cba214f83c59 kind: Event lastTimestamp: "2026-07-31T23:58:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:02Z" name: error-404-isvc-32541-predictor.18c78534629e248a namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: 5aae54ff-13fc-4ab5-b83f-1a95e7471396 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32541 namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: 42ac91ce-c7ec-4529-b2ef-62f501923e6b kind: Event lastTimestamp: "2026-07-31T23:57:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:36Z" name: error-404-isvc-32541.18c7852d6527d41f namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: caee189a-3ab5-43d8-85be-3b44ef24f609 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32541 namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 42ac91ce-c7ec-4529-b2ef-62f501923e6b kind: Event lastTimestamp: "2026-07-31T23:57:36Z" message: InferenceService [error-404-isvc-32541] is Ready metadata: creationTimestamp: "2026-07-31T23:57:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:36Z" name: error-404-isvc-32541.18c78538be059e98 namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: cb9aad86-1a20-4cd1-b6c7-f5d14d551d03 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T23:55:17.346749Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds to ip-10-0-128-75.ec2.internal metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T23:55:17Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c7851873f3bd85 namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: d6d68c83-acb4-4cc7-823b-6aee4e01f7fa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14450" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T23:55:17Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c785188e82f964 namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 7ba4b7e6-7acf-42a0-9c18-f67ca6a2e25c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1841" metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:17Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c78518903019b3 namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 843aa3ec-f7ae-43cb-a2d9-ad6c75c292fd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: "2026-07-31T23:55:36Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1841" in 18.259s (18.259s including waiting). Image size: 1333908588 bytes.' metadata: creationTimestamp: "2026-07-31T23:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:36Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c7851cd08b8fa8 namespace: kserve-ci-e2e-test resourceVersion: "14698" uid: 4947f331-bf9b-4da9-9ccb-88d2d68e2535 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: "2026-07-31T23:55:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T23:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:44Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c7851edfdfeed5 namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: 8e23550f-ac58-437a-974c-6a21d278d709 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: "2026-07-31T23:55:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T23:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:44Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c7851ee0d7aece namespace: kserve-ci-e2e-test resourceVersion: "14760" uid: c5746353-5c28-458a-b18d-7dd6baddab7f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: "2026-07-31T23:55:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-31T23:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:44Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c7851ee0e590b5 namespace: kserve-ci-e2e-test resourceVersion: "14761" uid: 569cc121-ccce-449e-a3b9-af8c5976e95d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: "2026-07-31T23:55:48Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.552s (3.552s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:48Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c7851fb4a14c42 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: aa160030-3645-4a8a-a619-5fc04932abd2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: "2026-07-31T23:55:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:48Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c7851fb9d37b8c namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: d2d09148-c782-413f-8458-e38f50bbcfb0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: "2026-07-31T23:55:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:48Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c7851fba47505b namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: fa73ad50-20bd-471b-9739-08512fb713fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: "2026-07-31T23:56:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:25Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c7851fbf67cac1 namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: d457aa25-a5b3-468c-bb0b-7c6aff38f4de reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: "2026-07-31T23:56:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:47Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c7852d60f12a28 namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: f1f0aa0c-bb3a-4a72-937f-1753f2848530 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: "2026-07-31T23:56:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:47Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c7852d60f473f8 namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: 98dd6e2e-aa94-44a7-806c-f9c9010ee843 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 4988fb9c-fe37-4d48-bd7b-6bea640dba66 kind: Event lastTimestamp: "2026-07-31T23:56:50Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:50Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds.18c7852e2fa05f14 namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: 3af66910-9b46-4ce4-98a1-2cb248e96e34 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-716dc-predictor-58d8b8d4f4 namespace: kserve-ci-e2e-test resourceVersion: "14432" uid: 03bd68ff-35e0-4f6d-8a3e-654262d52c28 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: 'Created pod: error-404-isvc-716dc-predictor-58d8b8d4f4-9xkds' metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:55:17Z" name: error-404-isvc-716dc-predictor-58d8b8d4f4.18c785187364079c namespace: kserve-ci-e2e-test resourceVersion: "14443" uid: c620c915-8a25-47b5-abcf-477d10ea992b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-716dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: 7d092595-a4dc-4a26-846f-cfa5d4c1b971 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Scaled up replica set error-404-isvc-716dc-predictor-58d8b8d4f4 from 0 to 1 metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:55:17Z" name: error-404-isvc-716dc-predictor.18c78518719167e2 namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: 2a3446b3-cbdc-4760-867d-8e5ea8a9eb95 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T23:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-716dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14431" uid: 7e309e90-fda4-4486-8972-40a5fdceb26e kind: Event lastTimestamp: "2026-07-31T23:56:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:32Z" name: error-404-isvc-716dc-predictor.18c7851bf01f9861 namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: b65a14d1-4c45-4802-a4c7-fa1af88ae5b9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T23:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-716dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14431" uid: 7e309e90-fda4-4486-8972-40a5fdceb26e kind: Event lastTimestamp: "2026-07-31T23:56:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:32Z" name: error-404-isvc-716dc-predictor.18c7851bf07e1c1d namespace: kserve-ci-e2e-test resourceVersion: "15198" uid: 10c790ee-a0b5-4f73-a3e8-78b665b75b8c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-716dc namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 3268e40b-f6ec-4b6a-b6be-7a97ede477e2 kind: Event lastTimestamp: "2026-07-31T23:56:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:56:35Z" name: error-404-isvc-716dc.18c785187196ed98 namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: 8174b5d3-6322-4d33-8e79-ce82642c0846 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-716dc namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 3268e40b-f6ec-4b6a-b6be-7a97ede477e2 kind: Event lastTimestamp: "2026-07-31T23:56:35Z" message: InferenceService [error-404-isvc-716dc] is Ready metadata: creationTimestamp: "2026-07-31T23:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:56:35Z" name: error-404-isvc-716dc.18c7852ab6bbc633 namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 070930ea-c848-45f8-9fd6-1087e6037107 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T23:57:37.326289Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: ff102fa8-a979-4023-8e8f-fd67bd0e15b9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr to ip-10-0-128-75.ec2.internal metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T23:57:37Z" name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr.18c785390b620f60 namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: 186226aa-2a58-478b-a627-99fed4cc5d4d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: ff102fa8-a979-4023-8e8f-fd67bd0e15b9 kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a3ab7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:37Z" name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr.18c785391820c3c3 namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 53c314eb-11fb-4608-9bd2-62752a5aa485 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr namespace: kserve-ci-e2e-test resourceVersion: "16108" uid: ff102fa8-a979-4023-8e8f-fd67bd0e15b9 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T23:57:38Z" name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr.18c7853948e880f6 namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: bec3fd13-e63a-4c25-8a5d-850f328f6012 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: ff102fa8-a979-4023-8e8f-fd67bd0e15b9 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:38Z" name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr.18c785394a3402ff namespace: kserve-ci-e2e-test resourceVersion: "16146" uid: 1a61be96-42a0-4609-9b36-919957d2d2b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: ff102fa8-a979-4023-8e8f-fd67bd0e15b9 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:38Z" name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr.18c785394d657af2 namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: 6b369718-1ddb-4e88-ab08-0fdb063ba9c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: ff102fa8-a979-4023-8e8f-fd67bd0e15b9 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:38Z" name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr.18c785394ddc4499 namespace: kserve-ci-e2e-test resourceVersion: "16150" uid: ae910961-07bd-4f86-912e-a31cb286ccd9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: ff102fa8-a979-4023-8e8f-fd67bd0e15b9 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:38Z" name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr.18c785394de7f34c namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 3e22c345-c64e-4735-9276-4a096e8c0c37 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: ff102fa8-a979-4023-8e8f-fd67bd0e15b9 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:38Z" name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr.18c78539510f3255 namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: 1be91c82-c25f-4b26-938b-58b9f3f40305 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: ff102fa8-a979-4023-8e8f-fd67bd0e15b9 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:38Z" name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr.18c78539518b29cf namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: a430173a-ed6f-4ad7-b65a-80548e6b2465 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T23:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: ff102fa8-a979-4023-8e8f-fd67bd0e15b9 kind: Event lastTimestamp: "2026-07-31T23:58:16Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:58:16Z" name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr.18c78539b33b6b43 namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 8f99fc95-15df-4d4c-91e2-6c353f5c3082 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: ff102fa8-a979-4023-8e8f-fd67bd0e15b9 kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:52Z" name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr.18c785ba3a16c8d3 namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: 9a90e83b-fbc0-4f42-98f2-3800accda251 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: ff102fa8-a979-4023-8e8f-fd67bd0e15b9 kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:52Z" name: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr.18c785ba3a17d894 namespace: kserve-ci-e2e-test resourceVersion: "20379" uid: c8fd3627-89eb-4b02-ae10-efb955ea9d1f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a3ab7-predictor-746489cd7b namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: 51c776ff-2e6e-41dc-bf9f-197a4c38d7b6 kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: 'Created pod: error-404-isvc-a3ab7-predictor-746489cd7b-qfpmr' metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:37Z" name: error-404-isvc-a3ab7-predictor-746489cd7b.18c785390af74dbd namespace: kserve-ci-e2e-test resourceVersion: "16101" uid: 023ca121-0140-4d7c-a6e8-34048efa15fc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a3ab7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: e9d76729-8d95-4f78-9c99-2b203c6d82a5 kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: Scaled up replica set error-404-isvc-a3ab7-predictor-746489cd7b from 0 to 1 metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:37Z" name: error-404-isvc-a3ab7-predictor.18c78539097dcd82 namespace: kserve-ci-e2e-test resourceVersion: "16095" uid: 78dc812e-b147-4dd0-8bd5-1b7ef409bb47 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T23:57:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3ab7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: 927ff83b-8447-40f3-a0ae-efa18dd0f5d7 kind: Event lastTimestamp: "2026-07-31T23:58:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:07Z" name: error-404-isvc-a3ab7-predictor.18c7853c87e9b67c namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: 8b071c35-59dc-4601-9d03-d3dc46f63b3b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T23:57:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3ab7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: 927ff83b-8447-40f3-a0ae-efa18dd0f5d7 kind: Event lastTimestamp: "2026-07-31T23:58:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:07Z" name: error-404-isvc-a3ab7-predictor.18c7853c884ae7b7 namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: c53343ce-6c9e-44e3-9e44-247426365176 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T23:58:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3ab7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 927ff83b-8447-40f3-a0ae-efa18dd0f5d7 kind: Event lastTimestamp: "2026-07-31T23:58:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:52Z" name: error-404-isvc-a3ab7-predictor.18c78543859d10ef namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: d5302252-4970-4458-840a-779ca42bf8bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T23:58:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3ab7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 927ff83b-8447-40f3-a0ae-efa18dd0f5d7 kind: Event lastTimestamp: "2026-07-31T23:58:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:52Z" name: error-404-isvc-a3ab7-predictor.18c7854385fe51e8 namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: cf0d627a-2792-491b-b30c-0ef1581bb70d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3ab7 namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 64375036-e7e0-4b51-94aa-9d44d125c692 kind: Event lastTimestamp: "2026-07-31T23:58:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:58:26Z" name: error-404-isvc-a3ab7.18c7853909b1f882 namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: f5fef488-5e58-4834-ac65-94f8609c41c6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3ab7 namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 64375036-e7e0-4b51-94aa-9d44d125c692 kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-a3ab7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a3ab7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:37Z" name: error-404-isvc-a3ab7.18c78539103e6569 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 1d919a51-9abc-44b0-b132-a313effd64aa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3ab7 namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 64375036-e7e0-4b51-94aa-9d44d125c692 kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a3ab7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:37Z" name: error-404-isvc-a3ab7.18c78539103ece97 namespace: kserve-ci-e2e-test resourceVersion: "16116" uid: 850139aa-d664-4bbd-8c1e-f5e0ca7be9ba reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:58:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3ab7 namespace: kserve-ci-e2e-test resourceVersion: "16609" uid: 64375036-e7e0-4b51-94aa-9d44d125c692 kind: Event lastTimestamp: "2026-07-31T23:58:26Z" message: InferenceService [error-404-isvc-a3ab7] is Ready metadata: creationTimestamp: "2026-07-31T23:58:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:58:26Z" name: error-404-isvc-a3ab7.18c785446e219cca namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 8b76d68f-5440-4a93-8f74-222b9374db2e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:17:17.546800Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv namespace: kserve-ci-e2e-test resourceVersion: "25937" uid: 12afa0f2-0926-4e52-a74d-bba84664f6c5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv to ip-10-0-128-75.ec2.internal metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:17:17Z" name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv.18c7864bd601de05 namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 9b8407aa-96f3-4760-8ec1-de2cfdfd65da reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 12afa0f2-0926-4e52-a74d-bba84664f6c5 kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-bb639-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:17Z" name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv.18c7864bdf768cd9 namespace: kserve-ci-e2e-test resourceVersion: "25959" uid: 09075c78-5282-4046-8fb7-d00583df0ee1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv namespace: kserve-ci-e2e-test resourceVersion: "25946" uid: 12afa0f2-0926-4e52-a74d-bba84664f6c5 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:17:18Z" name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv.18c7864c138ce399 namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 00dd4383-f821-4855-ae51-7770e1cf53d8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 12afa0f2-0926-4e52-a74d-bba84664f6c5 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:18Z" name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv.18c7864c14e18fea namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: 277f67a9-9c9c-4867-8eb0-392dc41bf121 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 12afa0f2-0926-4e52-a74d-bba84664f6c5 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:18Z" name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv.18c7864c182c3430 namespace: kserve-ci-e2e-test resourceVersion: "25982" uid: 09550a08-cf1b-4abb-973e-2b6d21dbb56b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 12afa0f2-0926-4e52-a74d-bba84664f6c5 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:18Z" name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv.18c7864c18a521a5 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 5d3ddcfa-eaec-49d6-8d69-bae374f2d99a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 12afa0f2-0926-4e52-a74d-bba84664f6c5 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:18Z" name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv.18c7864c18b006f0 namespace: kserve-ci-e2e-test resourceVersion: "25984" uid: 01d93340-45b5-4631-9469-56f408f69adc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 12afa0f2-0926-4e52-a74d-bba84664f6c5 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:18Z" name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv.18c7864c1cb53c9c namespace: kserve-ci-e2e-test resourceVersion: "25985" uid: 94e806df-05da-4e6b-8e9c-87a884ca769b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 12afa0f2-0926-4e52-a74d-bba84664f6c5 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:18Z" name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv.18c7864c1d2984b9 namespace: kserve-ci-e2e-test resourceVersion: "25986" uid: 0aa87598-0b0b-4ab7-a42b-fff5160713c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 12afa0f2-0926-4e52-a74d-bba84664f6c5 kind: Event lastTimestamp: "2026-08-01T00:17:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:56Z" name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv.18c7864c9022acba namespace: kserve-ci-e2e-test resourceVersion: "26513" uid: 2886b9a0-66e1-4178-aaf5-00ba3592364e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:34:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 12afa0f2-0926-4e52-a74d-bba84664f6c5 kind: Event lastTimestamp: "2026-08-01T00:34:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:34:37Z" name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv.18c7873ddeb56716 namespace: kserve-ci-e2e-test resourceVersion: "33618" uid: 0a125f37-19ab-47ac-88d5-31bbceef75cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:34:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 12afa0f2-0926-4e52-a74d-bba84664f6c5 kind: Event lastTimestamp: "2026-08-01T00:34:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:34:37Z" name: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv.18c7873ddeb5f984 namespace: kserve-ci-e2e-test resourceVersion: "33619" uid: 4f951113-7962-4b4b-b042-ea466499c340 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bb639-predictor-7bbf8d789b namespace: kserve-ci-e2e-test resourceVersion: "25929" uid: 0e02d717-4fbc-481f-93fc-9d58666efd00 kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: 'Created pod: error-404-isvc-bb639-predictor-7bbf8d789b-6tfmv' metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:17Z" name: error-404-isvc-bb639-predictor-7bbf8d789b.18c7864bd5a06e18 namespace: kserve-ci-e2e-test resourceVersion: "25939" uid: 21bc5712-2726-4b32-9edd-11aa0b40500c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bb639-predictor namespace: kserve-ci-e2e-test resourceVersion: "25927" uid: c94482aa-c554-4eea-8baf-e8b555916248 kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: Scaled up replica set error-404-isvc-bb639-predictor-7bbf8d789b from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:17Z" name: error-404-isvc-bb639-predictor.18c7864bd40467fd namespace: kserve-ci-e2e-test resourceVersion: "25930" uid: 881e7e89-407b-41f4-aaa3-f042c6476c7f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:17:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bb639-predictor namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: b255a47a-3965-4444-9b26-376759ac582b kind: Event lastTimestamp: "2026-08-01T00:18:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:02Z" name: error-404-isvc-bb639-predictor.18c7864f5285e6fd namespace: kserve-ci-e2e-test resourceVersion: "26559" uid: 216ddc16-c0e1-4380-8829-991577a2cda0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:17:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bb639-predictor namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: b255a47a-3965-4444-9b26-376759ac582b kind: Event lastTimestamp: "2026-08-01T00:18:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:02Z" name: error-404-isvc-bb639-predictor.18c7864f52e0af86 namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: 12ffc0f3-35c3-4d8c-b9f5-15b6536cce5e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:17:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bb639-predictor namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: b255a47a-3965-4444-9b26-376759ac582b kind: Event lastTimestamp: "2026-08-01T00:18:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:32Z" name: error-404-isvc-bb639-predictor.18c78652d180eb92 namespace: kserve-ci-e2e-test resourceVersion: "26871" uid: 5821c0ce-17b7-4b07-8e3e-245d47dcd29f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:17:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bb639-predictor namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: b255a47a-3965-4444-9b26-376759ac582b kind: Event lastTimestamp: "2026-08-01T00:18:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:32Z" name: error-404-isvc-bb639-predictor.18c78652d1e09005 namespace: kserve-ci-e2e-test resourceVersion: "26872" uid: bcb8bf96-e5cd-4e01-b466-7c9e9333f7c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bb639 namespace: kserve-ci-e2e-test resourceVersion: "25916" uid: bee24e19-a474-4dd0-b56c-bf63840010f8 kind: Event lastTimestamp: "2026-08-01T00:18:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:18:06Z" name: error-404-isvc-bb639.18c7864bd440e160 namespace: kserve-ci-e2e-test resourceVersion: "26610" uid: 70f38603-1f52-4c96-a59c-745d6f5113d3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bb639 namespace: kserve-ci-e2e-test resourceVersion: "25916" uid: bee24e19-a474-4dd0-b56c-bf63840010f8 kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: 'Failed to update status for InferenceService "error-404-isvc-bb639": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bb639": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:17:17Z" name: error-404-isvc-bb639.18c7864bd506bb6a namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: f7de241a-1bda-4756-8da7-f108cdad598a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bb639 namespace: kserve-ci-e2e-test resourceVersion: "25916" uid: bee24e19-a474-4dd0-b56c-bf63840010f8 kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bb639": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:17:17Z" name: error-404-isvc-bb639.18c7864bd50708f8 namespace: kserve-ci-e2e-test resourceVersion: "25938" uid: 679b9600-b1b0-4347-b03b-40cfb8b6175b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bb639 namespace: kserve-ci-e2e-test resourceVersion: "26608" uid: bee24e19-a474-4dd0-b56c-bf63840010f8 kind: Event lastTimestamp: "2026-08-01T00:18:06Z" message: InferenceService [error-404-isvc-bb639] is Ready metadata: creationTimestamp: "2026-08-01T00:18:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:18:06Z" name: error-404-isvc-bb639.18c786574b833d92 namespace: kserve-ci-e2e-test resourceVersion: "26609" uid: 31e0d256-6ba1-4f15-885e-d0d570c9b9fe reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:16:27.765464Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw namespace: kserve-ci-e2e-test resourceVersion: "25122" uid: 54cbebd2-2258-4358-a3f9-0cb5fc27f46c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d0074-predictor-64b97bfbdb-84kpw to ip-10-0-128-75.ec2.internal metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:16:27Z" name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw.18c786403eceeee6 namespace: kserve-ci-e2e-test resourceVersion: "25126" uid: 3efb7c82-d917-4fdc-a6ce-ad1285b0d188 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw namespace: kserve-ci-e2e-test resourceVersion: "25130" uid: 54cbebd2-2258-4358-a3f9-0cb5fc27f46c kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:16:28Z" name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw.18c786405898f915 namespace: kserve-ci-e2e-test resourceVersion: "25146" uid: 6aa9be03-003e-42eb-afca-43e6d18cc952 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: 54cbebd2-2258-4358-a3f9-0cb5fc27f46c kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:28Z" name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw.18c786405a43f962 namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: b07eecb4-93f3-41da-a29f-496fe8a38e93 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: 54cbebd2-2258-4358-a3f9-0cb5fc27f46c kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:28Z" name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw.18c786405d1df046 namespace: kserve-ci-e2e-test resourceVersion: "25155" uid: 291aefc9-e058-49c0-951f-b1c3f710a805 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: 54cbebd2-2258-4358-a3f9-0cb5fc27f46c kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:28Z" name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw.18c786405db83271 namespace: kserve-ci-e2e-test resourceVersion: "25156" uid: f7d8ace3-107b-43be-a008-a04a7e9c30a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: 54cbebd2-2258-4358-a3f9-0cb5fc27f46c kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:28Z" name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw.18c786405dc31fce namespace: kserve-ci-e2e-test resourceVersion: "25157" uid: 9cd5e2f9-8ce1-4c61-b594-a5fc8cf1779c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: 54cbebd2-2258-4358-a3f9-0cb5fc27f46c kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:28Z" name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw.18c78640614d0cb4 namespace: kserve-ci-e2e-test resourceVersion: "25159" uid: 4d6072fd-1601-4eb8-9097-00f34ce24406 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: 54cbebd2-2258-4358-a3f9-0cb5fc27f46c kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:28Z" name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw.18c7864061c450f5 namespace: kserve-ci-e2e-test resourceVersion: "25160" uid: 338aa5c1-9cdf-4015-9e44-43ec4728fac1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: 54cbebd2-2258-4358-a3f9-0cb5fc27f46c kind: Event lastTimestamp: "2026-08-01T00:17:14Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:14Z" name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw.18c7864067f5ac1a namespace: kserve-ci-e2e-test resourceVersion: "25809" uid: 97623842-3481-43c5-a593-f39592ec2a65 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: 54cbebd2-2258-4358-a3f9-0cb5fc27f46c kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw.18c786509a60e7b2 namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: f4544d05-d908-485b-8d0a-598dfec5d217 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: 54cbebd2-2258-4358-a3f9-0cb5fc27f46c kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw.18c786509a61ac79 namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 8ffb06ae-8dd1-4d1e-8d18-15a198144917 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: 54cbebd2-2258-4358-a3f9-0cb5fc27f46c kind: Event lastTimestamp: "2026-08-01T00:17:39Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:17:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:39Z" name: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw.18c78650f0303b7f namespace: kserve-ci-e2e-test resourceVersion: "26347" uid: ce62307d-422a-4c6d-9c14-8568c5f7a544 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d0074-predictor-64b97bfbdb namespace: kserve-ci-e2e-test resourceVersion: "25107" uid: 3750071e-a7ad-47ff-b87b-cac576f2d8d1 kind: Event lastTimestamp: "2026-08-01T00:16:27Z" message: 'Created pod: error-404-isvc-d0074-predictor-64b97bfbdb-84kpw' metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:16:27Z" name: error-404-isvc-d0074-predictor-64b97bfbdb.18c786403e609d77 namespace: kserve-ci-e2e-test resourceVersion: "25123" uid: f4862f78-9638-44c7-b7c9-1f20bd4b12c3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d0074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25102" uid: a391bf79-7ca0-479e-999f-38c54cc6b1cc kind: Event lastTimestamp: "2026-08-01T00:16:27Z" message: Scaled up replica set error-404-isvc-d0074-predictor-64b97bfbdb from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:16:27Z" name: error-404-isvc-d0074-predictor.18c786403cc7023b namespace: kserve-ci-e2e-test resourceVersion: "25111" uid: 57b58703-d6a5-4296-a764-586e36743261 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:16:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25108" uid: bfbc0461-c261-4e37-a4de-a66a3495ce94 kind: Event lastTimestamp: "2026-08-01T00:17:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:12Z" name: error-404-isvc-d0074-predictor.18c78643bb5b58c8 namespace: kserve-ci-e2e-test resourceVersion: "25796" uid: bcba5896-a839-452c-be8f-81086225b2f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:16:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25108" uid: bfbc0461-c261-4e37-a4de-a66a3495ce94 kind: Event lastTimestamp: "2026-08-01T00:17:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:12Z" name: error-404-isvc-d0074-predictor.18c78643bbba76e2 namespace: kserve-ci-e2e-test resourceVersion: "25797" uid: cd4a5c30-a3fd-4621-a936-6b1d821bfe50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25513" uid: bfbc0461-c261-4e37-a4de-a66a3495ce94 kind: Event lastTimestamp: "2026-08-01T00:17:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:27Z" name: error-404-isvc-d0074-predictor.18c7864e37a57d55 namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: 2b661232-b2a3-4fea-ba59-2a6781f87e6d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25513" uid: bfbc0461-c261-4e37-a4de-a66a3495ce94 kind: Event lastTimestamp: "2026-08-01T00:17:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:27Z" name: error-404-isvc-d0074-predictor.18c7864e38252ea1 namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: a7db3840-14a7-4568-8842-36bcc4251569 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0074 namespace: kserve-ci-e2e-test resourceVersion: "25081" uid: ace27065-2596-4f85-b1a2-b0b745ae4db8 kind: Event lastTimestamp: "2026-08-01T00:17:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:17:24Z" name: error-404-isvc-d0074.18c786403d04f48f namespace: kserve-ci-e2e-test resourceVersion: "26051" uid: 4f696724-0e01-4c3b-a7fb-d4d08ddbb3bf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0074 namespace: kserve-ci-e2e-test resourceVersion: "25081" uid: ace27065-2596-4f85-b1a2-b0b745ae4db8 kind: Event lastTimestamp: "2026-08-01T00:16:27Z" message: 'Failed to update status for InferenceService "error-404-isvc-d0074": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d0074": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:16:27Z" name: error-404-isvc-d0074.18c786403d70a2ea namespace: kserve-ci-e2e-test resourceVersion: "25117" uid: 5f6e67ed-f745-4a4d-9f9c-3d8d0a142895 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0074 namespace: kserve-ci-e2e-test resourceVersion: "25081" uid: ace27065-2596-4f85-b1a2-b0b745ae4db8 kind: Event lastTimestamp: "2026-08-01T00:16:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d0074": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:16:27Z" name: error-404-isvc-d0074.18c786403d70eb3c namespace: kserve-ci-e2e-test resourceVersion: "25119" uid: c560a110-4a7d-4f9f-9aed-a12fdf9c911c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0074 namespace: kserve-ci-e2e-test resourceVersion: "26049" uid: ace27065-2596-4f85-b1a2-b0b745ae4db8 kind: Event lastTimestamp: "2026-08-01T00:17:24Z" message: InferenceService [error-404-isvc-d0074] is Ready metadata: creationTimestamp: "2026-08-01T00:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:17:24Z" name: error-404-isvc-d0074.18c7864d772e4148 namespace: kserve-ci-e2e-test resourceVersion: "26050" uid: 33c3e0d2-3391-4aae-b4b3-24d747761112 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:08:02.603945Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d99eb-predictor-597b578c4-hvl99 namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: 63f601af-2cf4-4dd4-bf90-b7d2e98b97c1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d99eb-predictor-597b578c4-hvl99 to ip-10-0-128-75.ec2.internal metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:08:02Z" name: error-404-isvc-d99eb-predictor-597b578c4-hvl99.18c785caa0d5e636 namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: df5e8d61-724e-4600-ba40-d1a5446dd048 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d99eb-predictor-597b578c4-hvl99 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 63f601af-2cf4-4dd4-bf90-b7d2e98b97c1 kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:08:03Z" name: error-404-isvc-d99eb-predictor-597b578c4-hvl99.18c785cabaab5af6 namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: 38829fe0-a3c6-4877-8d3f-bcfb1bf23a3e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d99eb-predictor-597b578c4-hvl99 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 63f601af-2cf4-4dd4-bf90-b7d2e98b97c1 kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:03Z" name: error-404-isvc-d99eb-predictor-597b578c4-hvl99.18c785cabc19e665 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: b1b293e9-5353-4f83-9821-28fdaeb62daf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d99eb-predictor-597b578c4-hvl99 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 63f601af-2cf4-4dd4-bf90-b7d2e98b97c1 kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:03Z" name: error-404-isvc-d99eb-predictor-597b578c4-hvl99.18c785cabec38d3d namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 32717a5d-d97a-4283-aefc-fbda1d3f266f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d99eb-predictor-597b578c4-hvl99 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 63f601af-2cf4-4dd4-bf90-b7d2e98b97c1 kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:03Z" name: error-404-isvc-d99eb-predictor-597b578c4-hvl99.18c785cabf37b49d namespace: kserve-ci-e2e-test resourceVersion: "21415" uid: 7d2d80b7-5356-48a1-9b89-6ccc5efc018f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d99eb-predictor-597b578c4-hvl99 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 63f601af-2cf4-4dd4-bf90-b7d2e98b97c1 kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:03Z" name: error-404-isvc-d99eb-predictor-597b578c4-hvl99.18c785cabf42d0c0 namespace: kserve-ci-e2e-test resourceVersion: "21416" uid: e2efdff7-5eeb-4c5c-99fb-5afbb5b6faed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d99eb-predictor-597b578c4-hvl99 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 63f601af-2cf4-4dd4-bf90-b7d2e98b97c1 kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:03Z" name: error-404-isvc-d99eb-predictor-597b578c4-hvl99.18c785cac2666e65 namespace: kserve-ci-e2e-test resourceVersion: "21417" uid: 70f8e1d9-b8a7-4c56-8f14-69898738c525 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d99eb-predictor-597b578c4-hvl99 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 63f601af-2cf4-4dd4-bf90-b7d2e98b97c1 kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:03Z" name: error-404-isvc-d99eb-predictor-597b578c4-hvl99.18c785cac2ee9be6 namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: 128c1e78-548d-4cb9-b5fb-0aa10dc395d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-01T00:08:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d99eb-predictor-597b578c4-hvl99 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 63f601af-2cf4-4dd4-bf90-b7d2e98b97c1 kind: Event lastTimestamp: "2026-08-01T00:17:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:08:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:20Z" name: error-404-isvc-d99eb-predictor-597b578c4-hvl99.18c785cb13e39678 namespace: kserve-ci-e2e-test resourceVersion: "26007" uid: 76694884-d4c5-4b02-a7e9-e70a6af22b84 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d99eb-predictor-597b578c4-hvl99 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 63f601af-2cf4-4dd4-bf90-b7d2e98b97c1 kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:17Z" name: error-404-isvc-d99eb-predictor-597b578c4-hvl99.18c7864bcf2219e7 namespace: kserve-ci-e2e-test resourceVersion: "25907" uid: a135a9fa-9127-4283-88c4-e7be0349d2fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d99eb-predictor-597b578c4-hvl99 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 63f601af-2cf4-4dd4-bf90-b7d2e98b97c1 kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:17Z" name: error-404-isvc-d99eb-predictor-597b578c4-hvl99.18c7864bcf235717 namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: d65b65f6-e88e-43ca-9905-cf4ba6f72ca3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d99eb-predictor-597b578c4-hvl99 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 63f601af-2cf4-4dd4-bf90-b7d2e98b97c1 kind: Event lastTimestamp: "2026-08-01T00:17:20Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:20Z" name: error-404-isvc-d99eb-predictor-597b578c4-hvl99.18c7864c883e208f namespace: kserve-ci-e2e-test resourceVersion: "26006" uid: ff0f34ec-fe03-4d93-9e56-eac0fc5d0037 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d99eb-predictor-597b578c4 namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: b6c9399d-6ce9-4a52-97ae-30c1fce94418 kind: Event lastTimestamp: "2026-08-01T00:08:02Z" message: 'Created pod: error-404-isvc-d99eb-predictor-597b578c4-hvl99' metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:02Z" name: error-404-isvc-d99eb-predictor-597b578c4.18c785caa06d90f0 namespace: kserve-ci-e2e-test resourceVersion: "21385" uid: 8e496289-0a01-4e96-be4c-f0273ba86d3d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d99eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: ab95ae98-b85b-42a2-a902-40ce137d54e7 kind: Event lastTimestamp: "2026-08-01T00:08:02Z" message: Scaled up replica set error-404-isvc-d99eb-predictor-597b578c4 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:02Z" name: error-404-isvc-d99eb-predictor.18c785ca9eda3506 namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: 2853398c-2107-472d-9c90-dedf472caa87 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d99eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: a25b06c1-86c6-4489-bb0d-65614c7ff5ad kind: Event lastTimestamp: "2026-08-01T00:08:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:17Z" name: error-404-isvc-d99eb-predictor.18c785ce1d698cd8 namespace: kserve-ci-e2e-test resourceVersion: "21592" uid: a761c318-11c8-42ce-a948-74ed3e4ba261 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d99eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: a25b06c1-86c6-4489-bb0d-65614c7ff5ad kind: Event lastTimestamp: "2026-08-01T00:08:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:17Z" name: error-404-isvc-d99eb-predictor.18c785ce1dc2f0b2 namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: 2a25ec8c-ec09-46ce-aec4-47408612859c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-01T00:08:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d99eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21593" uid: a25b06c1-86c6-4489-bb0d-65614c7ff5ad kind: Event lastTimestamp: "2026-08-01T00:09:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:09:17Z" name: error-404-isvc-d99eb-predictor.18c785d19c7b280f namespace: kserve-ci-e2e-test resourceVersion: "22106" uid: 945717dc-a12b-41b1-91e2-8ec66353414e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-01T00:08:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d99eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21593" uid: a25b06c1-86c6-4489-bb0d-65614c7ff5ad kind: Event lastTimestamp: "2026-08-01T00:09:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:09:17Z" name: error-404-isvc-d99eb-predictor.18c785d19cdfa5d6 namespace: kserve-ci-e2e-test resourceVersion: "22107" uid: bc93ccc4-160b-4eaa-bf44-6364201f9f61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d99eb namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: 269c97b7-2b5c-4714-85ae-346e0c2fec2c kind: Event lastTimestamp: "2026-08-01T00:08:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:08:50Z" name: error-404-isvc-d99eb.18c785ca9ee216cc namespace: kserve-ci-e2e-test resourceVersion: "21867" uid: cfbd1e9b-b76a-4589-ac33-e9adb8452f21 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d99eb namespace: kserve-ci-e2e-test resourceVersion: "21864" uid: 269c97b7-2b5c-4714-85ae-346e0c2fec2c kind: Event lastTimestamp: "2026-08-01T00:08:50Z" message: InferenceService [error-404-isvc-d99eb] is Ready metadata: creationTimestamp: "2026-08-01T00:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:08:50Z" name: error-404-isvc-d99eb.18c785d5cf27572a namespace: kserve-ci-e2e-test resourceVersion: "21865" uid: e3b208fc-b8d9-4278-9265-9389bcdfea87 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:06:52.296155Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df362-predictor-6769b69b98-bnk4q namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: bcbd1dbf-5ca2-475a-a217-076476f448c3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-df362-predictor-6769b69b98-bnk4q to ip-10-0-128-75.ec2.internal metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:06:52Z" name: error-404-isvc-df362-predictor-6769b69b98-bnk4q.18c785ba422a2c39 namespace: kserve-ci-e2e-test resourceVersion: "20416" uid: d1dc6277-7890-4d0e-b722-260446cfab3d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df362-predictor-6769b69b98-bnk4q namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: bcbd1dbf-5ca2-475a-a217-076476f448c3 kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-df362-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:52Z" name: error-404-isvc-df362-predictor-6769b69b98-bnk4q.18c785ba495d93db namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: e35c1e61-942e-45a9-8669-acc5c8f5188e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df362-predictor-6769b69b98-bnk4q namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: bcbd1dbf-5ca2-475a-a217-076476f448c3 kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:06:53Z" name: error-404-isvc-df362-predictor-6769b69b98-bnk4q.18c785ba8033d986 namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 13e0f465-f3d4-4fd1-a705-a9131e4d5e82 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df362-predictor-6769b69b98-bnk4q namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: bcbd1dbf-5ca2-475a-a217-076476f448c3 kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:53Z" name: error-404-isvc-df362-predictor-6769b69b98-bnk4q.18c785ba81810fb1 namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 3f1492a0-ec10-4f78-90a1-562d450ba1ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df362-predictor-6769b69b98-bnk4q namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: bcbd1dbf-5ca2-475a-a217-076476f448c3 kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:53Z" name: error-404-isvc-df362-predictor-6769b69b98-bnk4q.18c785ba84925314 namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: 9d3fb03e-5ab7-4a88-9cec-127e189a3746 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df362-predictor-6769b69b98-bnk4q namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: bcbd1dbf-5ca2-475a-a217-076476f448c3 kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:53Z" name: error-404-isvc-df362-predictor-6769b69b98-bnk4q.18c785ba851135f4 namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: de1c8189-f81a-48d6-9bee-ebdc1253444b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df362-predictor-6769b69b98-bnk4q namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: bcbd1dbf-5ca2-475a-a217-076476f448c3 kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:53Z" name: error-404-isvc-df362-predictor-6769b69b98-bnk4q.18c785ba851dee3e namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 68a97116-991a-4b1a-a5c9-37dfed46c21a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df362-predictor-6769b69b98-bnk4q namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: bcbd1dbf-5ca2-475a-a217-076476f448c3 kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:53Z" name: error-404-isvc-df362-predictor-6769b69b98-bnk4q.18c785ba8830e529 namespace: kserve-ci-e2e-test resourceVersion: "20457" uid: f7661a28-c362-43f2-b0a5-62bbed4b4453 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df362-predictor-6769b69b98-bnk4q namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: bcbd1dbf-5ca2-475a-a217-076476f448c3 kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:53Z" name: error-404-isvc-df362-predictor-6769b69b98-bnk4q.18c785ba88bdfbd9 namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: 746d3b15-c71c-4a56-a5f2-25da13d28a08 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-01T00:06:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df362-predictor-6769b69b98-bnk4q namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: bcbd1dbf-5ca2-475a-a217-076476f448c3 kind: Event lastTimestamp: "2026-08-01T00:07:41Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:41Z" name: error-404-isvc-df362-predictor-6769b69b98-bnk4q.18c785baf1ebbaab namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: f05f53dc-8765-404e-8019-3b6000366fb1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df362-predictor-6769b69b98-bnk4q namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: bcbd1dbf-5ca2-475a-a217-076476f448c3 kind: Event lastTimestamp: "2026-08-01T00:08:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:02Z" name: error-404-isvc-df362-predictor-6769b69b98-bnk4q.18c785ca9a2c901b namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: c93c4ad1-7852-4b3d-ac85-51508d89722c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df362-predictor-6769b69b98-bnk4q namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: bcbd1dbf-5ca2-475a-a217-076476f448c3 kind: Event lastTimestamp: "2026-08-01T00:08:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:02Z" name: error-404-isvc-df362-predictor-6769b69b98-bnk4q.18c785ca9a2d3fba namespace: kserve-ci-e2e-test resourceVersion: "21360" uid: 35c8f3f5-3413-426c-b340-1caf6d77bda7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-df362-predictor-6769b69b98 namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: d0c5bd4c-ec3d-4e5b-851b-9c1131f6651b kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: 'Created pod: error-404-isvc-df362-predictor-6769b69b98-bnk4q' metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:06:52Z" name: error-404-isvc-df362-predictor-6769b69b98.18c785ba41c08de1 namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: e4b61cb7-c01e-4450-bbb9-b8718ed117c4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-df362-predictor namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: 28c5f930-c638-42fd-b264-68bbce213bfc kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: Scaled up replica set error-404-isvc-df362-predictor-6769b69b98 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:06:52Z" name: error-404-isvc-df362-predictor.18c785ba4010ec75 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: 349208b7-f5c2-479d-8aea-5270a34595b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:07:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df362-predictor namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 03e5ecbb-6479-4810-98fa-3270fb9b0b24 kind: Event lastTimestamp: "2026-08-01T00:07:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:22Z" name: error-404-isvc-df362-predictor.18c785bdbe9265f5 namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: 81439bb5-6835-4660-9629-9831a5871a0b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:07:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df362-predictor namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 03e5ecbb-6479-4810-98fa-3270fb9b0b24 kind: Event lastTimestamp: "2026-08-01T00:07:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:22Z" name: error-404-isvc-df362-predictor.18c785bdbef23240 namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: 0ec2ffc8-5db9-4120-b6ea-b8dba2dc2626 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:07:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df362-predictor namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 03e5ecbb-6479-4810-98fa-3270fb9b0b24 kind: Event lastTimestamp: "2026-08-01T00:07:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:52Z" name: error-404-isvc-df362-predictor.18c785c4bc88b77f namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: e337b3fc-6205-4f23-a28c-b9704007bef5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:07:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df362-predictor namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 03e5ecbb-6479-4810-98fa-3270fb9b0b24 kind: Event lastTimestamp: "2026-08-01T00:07:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:52Z" name: error-404-isvc-df362-predictor.18c785c4bce90198 namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: be676af4-bc5a-4a23-8585-9b809e1d3017 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df362 namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: a45c6173-df02-4f82-bd8d-01c445c1f557 kind: Event lastTimestamp: "2026-08-01T00:07:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:07:51Z" name: error-404-isvc-df362.18c785ba40173aa8 namespace: kserve-ci-e2e-test resourceVersion: "21151" uid: 9d2d2697-2a72-4bcf-b426-77731bcdeda8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df362 namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: a45c6173-df02-4f82-bd8d-01c445c1f557 kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: 'Failed to update status for InferenceService "error-404-isvc-df362": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-df362": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:06:52Z" name: error-404-isvc-df362.18c785ba40aa359d namespace: kserve-ci-e2e-test resourceVersion: "20409" uid: 0361360e-2073-4e2b-a7ae-ede9df948fad reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df362 namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: a45c6173-df02-4f82-bd8d-01c445c1f557 kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-df362": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:06:52Z" name: error-404-isvc-df362.18c785ba40aa81e8 namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: bd2ba435-2e95-41ae-9d8e-a6b9b44a2d1d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df362 namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: a45c6173-df02-4f82-bd8d-01c445c1f557 kind: Event lastTimestamp: "2026-08-01T00:07:51Z" message: InferenceService [error-404-isvc-df362] is Ready metadata: creationTimestamp: "2026-08-01T00:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:07:51Z" name: error-404-isvc-df362.18c785c801094705 namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: d1bd10fd-e660-4242-80eb-c56125fc9f29 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:06:02.460978Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9 namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: 576ba641-ea65-4b9e-8b4a-a08086f095cb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9 to ip-10-0-128-75.ec2.internal metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:06:02Z" name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9.18c785aea7c1b255 namespace: kserve-ci-e2e-test resourceVersion: "19887" uid: c293c318-8111-4435-86b2-4aa2f040d6fc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9 namespace: kserve-ci-e2e-test resourceVersion: "19892" uid: 576ba641-ea65-4b9e-8b4a-a08086f095cb kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:06:02Z" name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9.18c785aec21c3834 namespace: kserve-ci-e2e-test resourceVersion: "19908" uid: d31a53d0-6009-458e-bd34-5630fcd58c0e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 576ba641-ea65-4b9e-8b4a-a08086f095cb kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9.18c785aec3813ce1 namespace: kserve-ci-e2e-test resourceVersion: "19910" uid: 4bf3ec47-3f36-493a-b6e4-cad5a2e6d446 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 576ba641-ea65-4b9e-8b4a-a08086f095cb kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9.18c785aec70436e7 namespace: kserve-ci-e2e-test resourceVersion: "19913" uid: ab9414e9-73c6-4373-9339-fa535dc95898 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 576ba641-ea65-4b9e-8b4a-a08086f095cb kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9.18c785aec782c2fb namespace: kserve-ci-e2e-test resourceVersion: "19914" uid: 54111cdd-4a32-42a4-ba2a-3aab5f71c413 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 576ba641-ea65-4b9e-8b4a-a08086f095cb kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9.18c785aec78d3d06 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: 7ea74877-4126-45be-9acb-1fd62b091cd5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 576ba641-ea65-4b9e-8b4a-a08086f095cb kind: Event lastTimestamp: "2026-08-01T00:06:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:03Z" name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9.18c785aecaf1554e namespace: kserve-ci-e2e-test resourceVersion: "19920" uid: 0452a347-167d-42cc-bc79-30978a4360b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 576ba641-ea65-4b9e-8b4a-a08086f095cb kind: Event lastTimestamp: "2026-08-01T00:06:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:03Z" name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9.18c785aecb6b5aed namespace: kserve-ci-e2e-test resourceVersion: "19921" uid: 4d9d7579-0d9d-46a8-8c32-f7ccd5ca0a9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 576ba641-ea65-4b9e-8b4a-a08086f095cb kind: Event lastTimestamp: "2026-08-01T00:06:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:40Z" name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9.18c785af057684dc namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: 8a2298c6-7673-49f4-940b-f80914767026 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 576ba641-ea65-4b9e-8b4a-a08086f095cb kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:12Z" name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9.18c785befd797f34 namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 168dc815-aaa1-4530-8bd3-8b280c889b69 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 576ba641-ea65-4b9e-8b4a-a08086f095cb kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:12Z" name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9.18c785befd7a309e namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 93cc06d2-8091-4ea6-a8ed-8cfb15f04e45 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 576ba641-ea65-4b9e-8b4a-a08086f095cb kind: Event lastTimestamp: "2026-08-01T00:07:15Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:15Z" name: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9.18c785bf8d98e54d namespace: kserve-ci-e2e-test resourceVersion: "20825" uid: e8c3a8fa-071c-41e3-8a27-a59b94263517 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fe2ca-predictor-bb856dc98 namespace: kserve-ci-e2e-test resourceVersion: "19877" uid: 1953471d-5048-4faa-879e-44440adbf486 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: 'Created pod: error-404-isvc-fe2ca-predictor-bb856dc98-9pjb9' metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:06:02Z" name: error-404-isvc-fe2ca-predictor-bb856dc98.18c785aea75f7f5e namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: 69291818-7785-465d-a5dc-23f573d90b3a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fe2ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19875" uid: 369d6a1b-7513-4665-bea7-65f25f288df0 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Scaled up replica set error-404-isvc-fe2ca-predictor-bb856dc98 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:06:02Z" name: error-404-isvc-fe2ca-predictor.18c785aea5d140a3 namespace: kserve-ci-e2e-test resourceVersion: "19878" uid: 1a9688fb-e4e0-43bd-9742-a835b561c06f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fe2ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19876" uid: 1e4b00a2-89f4-47d3-8385-b5eadc7ce84b kind: Event lastTimestamp: "2026-08-01T00:06:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:06:47Z" name: error-404-isvc-fe2ca-predictor.18c785b2244f93da namespace: kserve-ci-e2e-test resourceVersion: "20264" uid: 2382e584-e2bb-46b9-8567-21d5ada5a6a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fe2ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19876" uid: 1e4b00a2-89f4-47d3-8385-b5eadc7ce84b kind: Event lastTimestamp: "2026-08-01T00:06:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:06:47Z" name: error-404-isvc-fe2ca-predictor.18c785b224b7aecc namespace: kserve-ci-e2e-test resourceVersion: "20265" uid: b45afbad-90df-4ebf-957b-755ae5a113d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fe2ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 1e4b00a2-89f4-47d3-8385-b5eadc7ce84b kind: Event lastTimestamp: "2026-08-01T00:07:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:02Z" name: error-404-isvc-fe2ca-predictor.18c785bca0bb834c namespace: kserve-ci-e2e-test resourceVersion: "20573" uid: 3136be9a-921e-4134-9bd0-6cabeced469a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fe2ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 1e4b00a2-89f4-47d3-8385-b5eadc7ce84b kind: Event lastTimestamp: "2026-08-01T00:07:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:02Z" name: error-404-isvc-fe2ca-predictor.18c785bca1255eba namespace: kserve-ci-e2e-test resourceVersion: "20576" uid: 6402218b-e61c-495b-9080-7150f6305271 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fe2ca namespace: kserve-ci-e2e-test resourceVersion: "19860" uid: 021ff483-7439-40d6-8b6c-4e85cc74c535 kind: Event lastTimestamp: "2026-08-01T00:06:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:06:50Z" name: error-404-isvc-fe2ca.18c785aea5e32875 namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: c11068e4-42a3-4094-a058-45fe13a3082b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fe2ca namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: 021ff483-7439-40d6-8b6c-4e85cc74c535 kind: Event lastTimestamp: "2026-08-01T00:06:50Z" message: InferenceService [error-404-isvc-fe2ca] is Ready metadata: creationTimestamp: "2026-08-01T00:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:06:50Z" name: error-404-isvc-fe2ca.18c785b9c180cd5f namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 108fe6bc-0dee-41b8-aeee-78f1814bf859 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T23:55:16.959435Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14353" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p to ip-10-0-128-75.ec2.internal metadata: creationTimestamp: "2026-07-31T23:55:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T23:55:16Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c785185cdde00b namespace: kserve-ci-e2e-test resourceVersion: "14360" uid: ce7dc3f6-0b23-42f6-8797-6350428b5002 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Add eth0 [10.134.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T23:55:17Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c785187711706a namespace: kserve-ci-e2e-test resourceVersion: "14461" uid: 8475c6ec-155a-45cc-acfd-b3cd65683178 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a7f8762eacdcae046df91671e5d778b36a484552558515ce8a80ad3a10a98099" already present on machine metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:17Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c785187893ec45 namespace: kserve-ci-e2e-test resourceVersion: "14468" uid: 272ba03b-249d-4947-a45a-c05d703e1af2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:17Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c785187b66b3af namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: 3d393f5c-c0ef-4fa5-a5d1-9f82ad683cba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:17Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c785187be320e0 namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: bdfd66b6-32b5-45ae-b4e3-227fb9d9dc7d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:55:23Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1841" metadata: creationTimestamp: "2026-07-31T23:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:23Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c78519e429dca1 namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: 3abbe746-b735-4f99-8b2a-f903978b9bc3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:55:44Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1841" in 21.203s (21.203s including waiting). Image size: 1555741111 bytes.' metadata: creationTimestamp: "2026-07-31T23:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:44Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c7851ed3fb33af namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 4b014b49-c61f-4222-aaaa-51da1fb48a3a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:55:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T23:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:44Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c7851ee360fd57 namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: e0b57202-c26d-49a2-a35e-b9377a76ec13 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:55:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T23:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:45Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c7851ee495a741 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 4c6243c2-1a0d-4d07-9789-f470e063f1ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:55:45Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-31T23:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:45Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c7851ee4a33fb4 namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: f085d253-affc-484d-bae3-f786420dd1b4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:55:48Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.524s (3.524s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:48Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c7851fb6bea71f namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: ad2ce99e-d92c-440f-be58-ec1339c38d59 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:55:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:48Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c7851fba4e4233 namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: 2447bbbf-e11e-45ad-a30a-8207fbf20d21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:55:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:48Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c7851fbab90393 namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: cd09d613-c86c-4ffe-ac1e-00cb39749180 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:56:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:55Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c7851fbf427826 namespace: kserve-ci-e2e-test resourceVersion: "15583" uid: ce343237-2c15-43da-82a8-b58db0826928 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:37Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c78539053e8d51 namespace: kserve-ci-e2e-test resourceVersion: "16076" uid: daf7c3c8-5665-4dd5-bad6-6a8154e891a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:37Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c78539053f033f namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: afc1b60b-0427-4d64-9136-27c02eb02084 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 76301075-df42-4218-b34e-47f97615bb7c kind: Event lastTimestamp: "2026-07-31T23:57:40Z" message: 'Readiness probe failed: Get "https://10.134.0.17:8643/healthz": dial tcp 10.134.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:40Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p.18c78539d3e4ae51 namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: 446a1667-ac39-42b5-9e4c-08e256fb3de7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-75774fc8c9 namespace: kserve-ci-e2e-test resourceVersion: "14347" uid: 2b527f77-fbbe-4f13-a37c-b29430cbd4bd kind: Event lastTimestamp: "2026-07-31T23:55:16Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-75774fc8c9-5mn9p' metadata: creationTimestamp: "2026-07-31T23:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:55:16Z" name: isvc-sklearn-graph-1-predictor-75774fc8c9.18c785185c6ccddd namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: 858ca014-40db-4b07-b3f8-ecec8ccf7401 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: e06da9e7-8af4-42e0-9358-af80281a0cd3 kind: Event lastTimestamp: "2026-07-31T23:55:16Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-75774fc8c9 from 0 to 1 metadata: creationTimestamp: "2026-07-31T23:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:55:16Z" name: isvc-sklearn-graph-1-predictor.18c78518585fda66 namespace: kserve-ci-e2e-test resourceVersion: "14348" uid: b1d802f7-388c-4d87-b52e-32ec8d5f81e7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T23:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: 21b0a1c6-37ce-4735-ae1b-dc45ae22f1d5 kind: Event lastTimestamp: "2026-07-31T23:56:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:32Z" name: isvc-sklearn-graph-1-predictor.18c7851bddac96c4 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: 1a2d5021-faf7-4a8f-b4cf-2a87a5bc6cc0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T23:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: 21b0a1c6-37ce-4735-ae1b-dc45ae22f1d5 kind: Event lastTimestamp: "2026-07-31T23:56:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:32Z" name: isvc-sklearn-graph-1-predictor.18c7851bde506b7f namespace: kserve-ci-e2e-test resourceVersion: "15191" uid: d172bff9-39e2-4094-8354-991692a67df2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T23:56:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 21b0a1c6-37ce-4735-ae1b-dc45ae22f1d5 kind: Event lastTimestamp: "2026-07-31T23:57:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:32Z" name: isvc-sklearn-graph-1-predictor.18c785265b92b8e2 namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 43eefeb0-6ee1-4808-b3bb-6e7ae1614f64 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T23:56:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 21b0a1c6-37ce-4735-ae1b-dc45ae22f1d5 kind: Event lastTimestamp: "2026-07-31T23:57:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:32Z" name: isvc-sklearn-graph-1-predictor.18c785265bf72ac7 namespace: kserve-ci-e2e-test resourceVersion: "15926" uid: 8e978fe6-50cf-455f-b287-d057e6dfd6f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-31T23:55:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: f538d463-bab0-4cb6-ba4b-aabdae326e41 kind: Event lastTimestamp: "2026-07-31T23:57:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T23:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:05Z" name: isvc-sklearn-graph-1.18c785185e8f3d67 namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: 4dc244b1-c3f6-4491-b7ad-8f3f891e8e97 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: f538d463-bab0-4cb6-ba4b-aabdae326e41 kind: Event lastTimestamp: "2026-07-31T23:57:05Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-31T23:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:05Z" name: isvc-sklearn-graph-1.18c78531b2c9fb7e namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: 77d19503-13f0-47b1-8e12-36c62464dfdf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T23:55:17.455090Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14470" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t to ip-10-0-128-75.ec2.internal metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T23:55:17Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c785187a68edb5 namespace: kserve-ci-e2e-test resourceVersion: "14476" uid: 9222fb38-1c9d-4749-9c98-351fb3e29541 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T23:55:17Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c7851894ec041d namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: 1afa7e3f-b0eb-4424-bbe9-8afd2ddfb0f1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a7f8762eacdcae046df91671e5d778b36a484552558515ce8a80ad3a10a98099" already present on machine metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:17Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c78518964ff6f1 namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 437da7aa-f027-4f4e-9e95-365fa7f855b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:17Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c7851899476d2c namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: ecb0e459-93d3-4c21-86c4-4a967b682207 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:17Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c7851899ba5a49 namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: a9e37d22-939a-43b0-9a94-fdab5ed790ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:55:23Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1841" metadata: creationTimestamp: "2026-07-31T23:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:23Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c78519e44c00bc namespace: kserve-ci-e2e-test resourceVersion: "14577" uid: a0ede663-9e3e-40f4-a1d3-0a37cb93f46a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:55:36Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1841" in 12.668s (12.668s including waiting). Image size: 1555741111 bytes.' metadata: creationTimestamp: "2026-07-31T23:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:36Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c7851cd761fb0d namespace: kserve-ci-e2e-test resourceVersion: "14699" uid: bd90db76-5157-42a2-8a35-a99eef7c4620 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:55:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T23:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:44Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c7851ee0ba0d4c namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: 4fe2ddb7-f5af-46e6-8377-ffe5b7ac696b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:55:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T23:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:44Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c7851ee267ac32 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: f99d36f1-57eb-4995-b049-1f00c481717f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:55:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-31T23:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:44Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c7851ee2740cfb namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: 1a80f53e-3a25-439c-93e7-e969e6a71634 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:55:48Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.543s (3.543s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:48Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c7851fb5ab8150 namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: 4ad683d0-f762-4bdc-950c-35690472edf8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:55:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:48Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c7851fba659692 namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 35110b00-b11f-4926-b2ce-b979cb382b81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:55:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:48Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c7851fbb2153fb namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 9fbcdb48-91bb-4bb9-b6fe-a0a88fcd499d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:56:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:54Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c7851fbf9001ea namespace: kserve-ci-e2e-test resourceVersion: "15564" uid: b5d0852c-25af-46c7-b765-3c5c21702e96 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:37Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c7853901a32c57 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 14e7518e-69f8-4c91-9e8b-08baf7f56dc8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:37Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c7853901a3b333 namespace: kserve-ci-e2e-test resourceVersion: "16050" uid: f11e41a2-1d09-476d-be84-63471ad7c101 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f0cf1575-e02f-4cdb-9810-c55d478f4d54 kind: Event lastTimestamp: "2026-07-31T23:57:39Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:39Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t.18c785399805ad82 namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 7242d9ea-e74a-4d6f-9645-659439bf976b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-76b49f6c9c namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: 52c0b5a2-5c6a-4465-af6b-82446a5fbed1 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-76b49f6c9c-7c92t' metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:55:17Z" name: isvc-sklearn-graph-2-predictor-76b49f6c9c.18c7851879dbdf60 namespace: kserve-ci-e2e-test resourceVersion: "14471" uid: 4b502478-919d-4455-9feb-05ec781c4680 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: ceaf32dc-29d0-46f2-9a60-ee8c3290c66a kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-76b49f6c9c from 0 to 1 metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:55:17Z" name: isvc-sklearn-graph-2-predictor.18c7851877460196 namespace: kserve-ci-e2e-test resourceVersion: "14463" uid: d0b10587-70f0-43d0-b212-a7e914e9bac5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T23:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 8c2aafb5-2711-47a6-8a84-0b0ed7d2719b kind: Event lastTimestamp: "2026-07-31T23:56:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:32Z" name: isvc-sklearn-graph-2-predictor.18c7851bf5ca704d namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: ae6a71e9-5901-4c1d-a7d9-b04a773a805d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T23:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 8c2aafb5-2711-47a6-8a84-0b0ed7d2719b kind: Event lastTimestamp: "2026-07-31T23:56:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:32Z" name: isvc-sklearn-graph-2-predictor.18c7851bf633a739 namespace: kserve-ci-e2e-test resourceVersion: "15200" uid: 01307780-17bf-424d-9457-4c3229f2054c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14666" uid: 8c2aafb5-2711-47a6-8a84-0b0ed7d2719b kind: Event lastTimestamp: "2026-07-31T23:57:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:32Z" name: isvc-sklearn-graph-2-predictor.18c7852d700183a9 namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 7efb5ad3-9e83-4e10-9bcc-7489b13ce011 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14666" uid: 8c2aafb5-2711-47a6-8a84-0b0ed7d2719b kind: Event lastTimestamp: "2026-07-31T23:57:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:32Z" name: isvc-sklearn-graph-2-predictor.18c7852d70855f62 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 490cf87d-6bcb-4985-9d11-f51bded3c97b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 938d9257-66ff-4116-9624-cd819b48242b kind: Event lastTimestamp: "2026-07-31T23:57:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:04Z" name: isvc-sklearn-graph-2.18c7851877435606 namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 47140dc9-0493-4bd6-b693-9368e39b8222 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: 938d9257-66ff-4116-9624-cd819b48242b kind: Event lastTimestamp: "2026-07-31T23:57:04Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-31T23:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:04Z" name: isvc-sklearn-graph-2.18c7853176ff731e namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: e339cdbf-66b9-409f-a003-317e31ad8585 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T23:55:17.260531Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-skw2q to ip-10-0-128-75.ec2.internal metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T23:55:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c785186ed02e8f namespace: kserve-ci-e2e-test resourceVersion: "14420" uid: a3cc48ee-0d43-4a6f-bc75-ec362e736540 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14425" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Add eth0 [10.134.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T23:55:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c785188845b2e5 namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 2fa575fd-9da7-4169-91e5-66f0c9e0a619 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a7f8762eacdcae046df91671e5d778b36a484552558515ce8a80ad3a10a98099" already present on machine metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c7851889940c45 namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: 5e1f44b5-3865-4fbf-8315-112f2b641baa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c785188c6e77d6 namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: 0af85317-8e31-4535-9103-b0040fa23b5a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c785188d0949fb namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 2dbdb291-15c8-47a1-93cc-8a76a330d0f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:55:22Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-31T23:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c78519a822fac1 namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 8e4939be-acfb-4057-8291-9943705f72ac reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:55:45Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 22.772s (22.772s including waiting). Image size: 1307355894 bytes.' metadata: creationTimestamp: "2026-07-31T23:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c7851ef57bc428 namespace: kserve-ci-e2e-test resourceVersion: "14769" uid: d7359228-3058-4042-9f88-552e8219847c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:55:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T23:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c7851efc17c1cd namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: 5f22ea6f-f358-4e32-8985-5869dc195aac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:55:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T23:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c7851efdaa8566 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: db6c313c-a46f-423e-adc7-c327c1f590aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:55:45Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-31T23:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c7851efdbd995b namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 7916d84c-ae4d-45f8-b8bc-7804f2f69efc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:55:48Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.07s (3.07s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c7851fb4c824db namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: 1af96a90-ef65-4e29-b9f8-27669fe42c23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:55:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c7851fba57ba35 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 238ba5b3-452a-4947-bae7-c17458ad2658 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:55:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c7851fbad166ca namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: a149f9b3-16f2-4974-bd49-7f02744d0f2d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-31T23:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:56:44Z" message: 'Readiness probe failed: dial tcp 10.134.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c7851fbfad4c44 namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: 3f03d4a3-c8f7-472f-b672-8057d2003df0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c78538fed99784 namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 1213a5c2-968b-400a-af7c-3add3412b175 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c78538feda2286 namespace: kserve-ci-e2e-test resourceVersion: "16028" uid: b0ff3f48-f302-4f16-af09-b83dea557a23 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-skw2q namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: b7cd0c34-4fac-4008-b972-54541f103f61 kind: Event lastTimestamp: "2026-07-31T23:57:39Z" message: 'Readiness probe failed: Get "https://10.134.0.18:8643/healthz": dial tcp 10.134.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-skw2q.18c785399801dff1 namespace: kserve-ci-e2e-test resourceVersion: "16169" uid: 853c6a30-768d-49f1-9d3f-3579fd45338e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-75.ec2.internal source: component: kubelet host: ip-10-0-128-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14405" uid: a3100ac2-1398-4551-9024-88e18fd5ee61 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-skw2q' metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:55:17Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c785186e3ab27e namespace: kserve-ci-e2e-test resourceVersion: "14417" uid: 4f0923f9-a420-4139-b43d-5c3a0fdcfb35 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14402" uid: c4d5584c-ac07-48bb-933c-e248683e3025 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:55:17Z" name: isvc-xgboost-graph-predictor.18c785186b77e613 namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: 570e11c2-e111-48b0-8d17-fadcfbe0a349 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T23:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: b3b7a420-bbf1-452f-9bd9-02c9f76a0076 kind: Event lastTimestamp: "2026-07-31T23:56:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:32Z" name: isvc-xgboost-graph-predictor.18c7851bea24318f namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: ac4ae511-262c-4d9c-8bd1-501bab36511a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T23:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: b3b7a420-bbf1-452f-9bd9-02c9f76a0076 kind: Event lastTimestamp: "2026-07-31T23:56:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:32Z" name: isvc-xgboost-graph-predictor.18c7851bea8421b7 namespace: kserve-ci-e2e-test resourceVersion: "15196" uid: 80d5c933-1c4b-492f-a8b6-9c4b48f99e77 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: b3b7a420-bbf1-452f-9bd9-02c9f76a0076 kind: Event lastTimestamp: "2026-07-31T23:57:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:32Z" name: isvc-xgboost-graph-predictor.18c7852d642efd06 namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: e1683b46-7095-4fbf-871d-1c41f5d8d710 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: b3b7a420-bbf1-452f-9bd9-02c9f76a0076 kind: Event lastTimestamp: "2026-07-31T23:57:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:32Z" name: isvc-xgboost-graph-predictor.18c7852d64a707d9 namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: bca9dd2c-daa9-49b0-afbb-2ad23aa34475 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14387" uid: e3d9a1ae-ee3d-4c18-a960-caebaa99e24c kind: Event lastTimestamp: "2026-07-31T23:56:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:56:54Z" name: isvc-xgboost-graph.18c785186bc86931 namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: a9a59f09-ac34-472a-a214-07535bcaadfa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: e3d9a1ae-ee3d-4c18-a960-caebaa99e24c kind: Event lastTimestamp: "2026-07-31T23:56:54Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-31T23:56:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:56:54Z" name: isvc-xgboost-graph.18c7852f2305222e namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: ab671dd4-4280-4e2b-8db3-fd34cbd34cab reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T23:57:26.901724Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-65d5b44868-4ffg2 namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: e33a2914-6477-4923-8476-3987cd98783a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-65d5b44868-4ffg2 to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-07-31T23:57:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T23:57:26Z" name: model-chainer-65d5b44868-4ffg2.18c785369e07ce36 namespace: kserve-ci-e2e-test resourceVersion: "15871" uid: 0db599b0-d707-4260-8292-1b767e64bd84 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:27Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-65d5b44868-4ffg2 namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: e33a2914-6477-4923-8476-3987cd98783a kind: Event lastTimestamp: "2026-07-31T23:57:27Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T23:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T23:57:27Z" name: model-chainer-65d5b44868-4ffg2.18c78536b7e1adfb namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: 35703210-98e3-43e7-b189-7642e630ea59 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-65d5b44868-4ffg2 namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: e33a2914-6477-4923-8476-3987cd98783a kind: Event lastTimestamp: "2026-07-31T23:57:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:56dbaa30789264fbe80634bdc08e7f43bcfc19e5006180ec98cfd4cb6b8d99d6" already present on machine metadata: creationTimestamp: "2026-07-31T23:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:27Z" name: model-chainer-65d5b44868-4ffg2.18c78536b91043c4 namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: 9bebdf84-331b-4297-8f6f-ada2662ec7a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-65d5b44868-4ffg2 namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: e33a2914-6477-4923-8476-3987cd98783a kind: Event lastTimestamp: "2026-07-31T23:57:27Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-31T23:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:27Z" name: model-chainer-65d5b44868-4ffg2.18c78536bc81f839 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: f1f435ec-d33b-4891-8499-3aa55e136408 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-65d5b44868-4ffg2 namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: e33a2914-6477-4923-8476-3987cd98783a kind: Event lastTimestamp: "2026-07-31T23:57:27Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-31T23:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:27Z" name: model-chainer-65d5b44868-4ffg2.18c78536bd0d7806 namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: d3bfee23-3488-4a3c-95d0-896af47706cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-65d5b44868-4ffg2 namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: e33a2914-6477-4923-8476-3987cd98783a kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:37Z" name: model-chainer-65d5b44868-4ffg2.18c78538f895a2e6 namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 77bda830-4b62-4138-b629-bd58761fe11a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-65d5b44868-4ffg2 namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: e33a2914-6477-4923-8476-3987cd98783a kind: Event lastTimestamp: "2026-07-31T23:58:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:58:03Z" name: model-chainer-65d5b44868-4ffg2.18c78539577f28a7 namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 8b7da115-de06-45b9-a5ca-115cfe77d7ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-65d5b44868 namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: a33dc056-e850-48bb-80d0-b4c2c16d6176 kind: Event lastTimestamp: "2026-07-31T23:57:26Z" message: 'Created pod: model-chainer-65d5b44868-4ffg2' metadata: creationTimestamp: "2026-07-31T23:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:26Z" name: model-chainer-65d5b44868.18c785369d7de2d0 namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: 6684ca4e-0763-4db3-af11-7a28e4e042c1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: ecd5b5db-e287-4307-a68f-d4cc52e1e528 kind: Event lastTimestamp: "2026-07-31T23:57:26Z" message: Scaled up replica set model-chainer-65d5b44868 from 0 to 1 metadata: creationTimestamp: "2026-07-31T23:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:26Z" name: model-chainer.18c785369cc41a4e namespace: kserve-ci-e2e-test resourceVersion: "15864" uid: caeb4e76-fbf8-41bb-8b93-2310401b3016 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: 98b7c0c5-865b-4b12-a2fb-83f4174a37a3 kind: Event lastTimestamp: "2026-07-31T23:57:26Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-31T23:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:26Z" name: model-chainer.18c78536a1f65505 namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 4c545f01-b498-4682-a6a3-004ff7ff34a1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T23:58:37.259618Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a3ab7-5cc8864447-tbfbv namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: fcfc7c13-06ba-4fb4-a473-c3323bc9e4b8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a3ab7-5cc8864447-tbfbv to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-07-31T23:58:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T23:58:37Z" name: sequence-graph-a3ab7-5cc8864447-tbfbv.18c78546ffb01667 namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: e5cfe540-0341-4491-830b-a00c1ee511e8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:58:37Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a3ab7-5cc8864447-tbfbv namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: fcfc7c13-06ba-4fb4-a473-c3323bc9e4b8 kind: Event lastTimestamp: "2026-07-31T23:58:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-a3ab7-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T23:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:58:37Z" name: sequence-graph-a3ab7-5cc8864447-tbfbv.18c7854709afcb38 namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 83f5f9af-f99d-43b8-b2cc-4a1a45d24f62 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:58:38Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a3ab7-5cc8864447-tbfbv namespace: kserve-ci-e2e-test resourceVersion: "16708" uid: fcfc7c13-06ba-4fb4-a473-c3323bc9e4b8 kind: Event lastTimestamp: "2026-07-31T23:58:38Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T23:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T23:58:38Z" name: sequence-graph-a3ab7-5cc8864447-tbfbv.18c785473d562c32 namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: 522434a4-d780-42f4-9954-a348c95837ef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:58:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3ab7} kind: Pod name: sequence-graph-a3ab7-5cc8864447-tbfbv namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: fcfc7c13-06ba-4fb4-a473-c3323bc9e4b8 kind: Event lastTimestamp: "2026-07-31T23:58:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:56dbaa30789264fbe80634bdc08e7f43bcfc19e5006180ec98cfd4cb6b8d99d6" already present on machine metadata: creationTimestamp: "2026-07-31T23:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:58:38Z" name: sequence-graph-a3ab7-5cc8864447-tbfbv.18c785473e9def62 namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: e5d76c15-a4b6-41f4-b551-87b19f9b69ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:58:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3ab7} kind: Pod name: sequence-graph-a3ab7-5cc8864447-tbfbv namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: fcfc7c13-06ba-4fb4-a473-c3323bc9e4b8 kind: Event lastTimestamp: "2026-07-31T23:58:38Z" message: 'Created container: sequence-graph-a3ab7' metadata: creationTimestamp: "2026-07-31T23:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:58:38Z" name: sequence-graph-a3ab7-5cc8864447-tbfbv.18c78547420a8663 namespace: kserve-ci-e2e-test resourceVersion: "16733" uid: 2e515def-e59d-4097-9fc5-f1cd21a9bdc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:58:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3ab7} kind: Pod name: sequence-graph-a3ab7-5cc8864447-tbfbv namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: fcfc7c13-06ba-4fb4-a473-c3323bc9e4b8 kind: Event lastTimestamp: "2026-07-31T23:58:38Z" message: Started container sequence-graph-a3ab7 metadata: creationTimestamp: "2026-07-31T23:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:58:38Z" name: sequence-graph-a3ab7-5cc8864447-tbfbv.18c785474280b057 namespace: kserve-ci-e2e-test resourceVersion: "16734" uid: 628073e3-e7c0-4fde-8460-e2fb0abe2ffc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3ab7} kind: Pod name: sequence-graph-a3ab7-5cc8864447-tbfbv namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: fcfc7c13-06ba-4fb4-a473-c3323bc9e4b8 kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: Stopping container sequence-graph-a3ab7 metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:52Z" name: sequence-graph-a3ab7-5cc8864447-tbfbv.18c785ba30ce98b9 namespace: kserve-ci-e2e-test resourceVersion: "20333" uid: 227f01ff-0572-4995-b191-0193751f228d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3ab7} kind: Pod name: sequence-graph-a3ab7-5cc8864447-tbfbv namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: fcfc7c13-06ba-4fb4-a473-c3323bc9e4b8 kind: Event lastTimestamp: "2026-08-01T00:07:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-01T00:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:19Z" name: sequence-graph-a3ab7-5cc8864447-tbfbv.18c785bae0054f58 namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: 53b55265-6d84-4e93-b760-1cf9537a3958 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:58:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a3ab7-5cc8864447 namespace: kserve-ci-e2e-test resourceVersion: "16694" uid: 82d1d52e-85c9-47fa-ae68-05d00252a380 kind: Event lastTimestamp: "2026-07-31T23:58:37Z" message: 'Created pod: sequence-graph-a3ab7-5cc8864447-tbfbv' metadata: creationTimestamp: "2026-07-31T23:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:37Z" name: sequence-graph-a3ab7-5cc8864447.18c78546fed9798b namespace: kserve-ci-e2e-test resourceVersion: "16702" uid: e092f76a-ffee-42df-91f7-c52387104f27 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:58:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a3ab7 namespace: kserve-ci-e2e-test resourceVersion: "16692" uid: 520cfc1e-9eaf-4d6c-8db2-bba69694d951 kind: Event lastTimestamp: "2026-07-31T23:58:37Z" message: Scaled up replica set sequence-graph-a3ab7-5cc8864447 from 0 to 1 metadata: creationTimestamp: "2026-07-31T23:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:37Z" name: sequence-graph-a3ab7.18c78546fdbc366d namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 939a6a90-6696-4df6-89ee-b53c5f2c0e38 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:58:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a3ab7 namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: 7fe30ef7-0a5c-410a-9b6c-c0f0476e0182 kind: Event lastTimestamp: "2026-07-31T23:58:37Z" message: InferenceGraph [sequence-graph-a3ab7] is Ready metadata: creationTimestamp: "2026-07-31T23:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:58:37Z" name: sequence-graph-a3ab7.18c78547045788b7 namespace: kserve-ci-e2e-test resourceVersion: "16712" uid: d71d3ea2-4de9-4070-8596-80995d5d274b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T23:58:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a3ab7 namespace: kserve-ci-e2e-test resourceVersion: "16693" uid: 25372980-0889-4447-8409-a2b5fafe81a2 kind: Event lastTimestamp: "2026-07-31T23:59:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:59:07Z" name: sequence-graph-a3ab7.18c7854a7c4fc590 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 5ed23e6d-57ad-48e3-a9d8-25e86e13f863 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T23:58:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a3ab7 namespace: kserve-ci-e2e-test resourceVersion: "16693" uid: 25372980-0889-4447-8409-a2b5fafe81a2 kind: Event lastTimestamp: "2026-07-31T23:59:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:59:07Z" name: sequence-graph-a3ab7.18c7854a7cb6f35a namespace: kserve-ci-e2e-test resourceVersion: "16949" uid: 153ccbe6-bc72-432d-b4e7-5902a9c19bcd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:09:02.573948Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d99eb-8bf569c97-62v27 namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: fa70dcc9-b8a7-4b3c-a582-290cbec06a9a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-d99eb-8bf569c97-62v27 to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:09:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:09:02Z" name: sequence-graph-d99eb-8bf569c97-62v27.18c785d8975381a3 namespace: kserve-ci-e2e-test resourceVersion: "21978" uid: f5bc2284-87ad-401d-8a0b-96312eb7e8e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:09:03Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d99eb-8bf569c97-62v27 namespace: kserve-ci-e2e-test resourceVersion: "21981" uid: fa70dcc9-b8a7-4b3c-a582-290cbec06a9a kind: Event lastTimestamp: "2026-08-01T00:09:03Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:09:03Z" name: sequence-graph-d99eb-8bf569c97-62v27.18c785d8b136eb45 namespace: kserve-ci-e2e-test resourceVersion: "21993" uid: 8d942eb4-075b-4baf-a09b-0dd2cca9fad0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:09:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d99eb} kind: Pod name: sequence-graph-d99eb-8bf569c97-62v27 namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: fa70dcc9-b8a7-4b3c-a582-290cbec06a9a kind: Event lastTimestamp: "2026-08-01T00:09:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:56dbaa30789264fbe80634bdc08e7f43bcfc19e5006180ec98cfd4cb6b8d99d6" already present on machine metadata: creationTimestamp: "2026-08-01T00:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:09:03Z" name: sequence-graph-d99eb-8bf569c97-62v27.18c785d8b31fe46e namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: 37cc5bad-4239-4e31-8700-26cd6d038c6d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:09:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d99eb} kind: Pod name: sequence-graph-d99eb-8bf569c97-62v27 namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: fa70dcc9-b8a7-4b3c-a582-290cbec06a9a kind: Event lastTimestamp: "2026-08-01T00:09:03Z" message: 'Created container: sequence-graph-d99eb' metadata: creationTimestamp: "2026-08-01T00:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:09:03Z" name: sequence-graph-d99eb-8bf569c97-62v27.18c785d8b660fbc5 namespace: kserve-ci-e2e-test resourceVersion: "21998" uid: 94504fd0-2280-49ab-9799-5e522bac799a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:09:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d99eb} kind: Pod name: sequence-graph-d99eb-8bf569c97-62v27 namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: fa70dcc9-b8a7-4b3c-a582-290cbec06a9a kind: Event lastTimestamp: "2026-08-01T00:09:03Z" message: Started container sequence-graph-d99eb metadata: creationTimestamp: "2026-08-01T00:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:09:03Z" name: sequence-graph-d99eb-8bf569c97-62v27.18c785d8b6d5f354 namespace: kserve-ci-e2e-test resourceVersion: "21999" uid: 72746e5a-aa70-4510-82be-b69c931343ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d99eb} kind: Pod name: sequence-graph-d99eb-8bf569c97-62v27 namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: fa70dcc9-b8a7-4b3c-a582-290cbec06a9a kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: Stopping container sequence-graph-d99eb metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:17Z" name: sequence-graph-d99eb-8bf569c97-62v27.18c7864bc4e2954f namespace: kserve-ci-e2e-test resourceVersion: "25857" uid: fa1b112d-c41d-4814-bdd5-8e2d1efd47a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:17:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d99eb} kind: Pod name: sequence-graph-d99eb-8bf569c97-62v27 namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: fa70dcc9-b8a7-4b3c-a582-290cbec06a9a kind: Event lastTimestamp: "2026-08-01T00:17:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-01T00:17:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:44Z" name: sequence-graph-d99eb-8bf569c97-62v27.18c7864c62c9ca1b namespace: kserve-ci-e2e-test resourceVersion: "26399" uid: 90a8c1c6-04ea-4123-a5dc-16050e4cf2cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:09:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-d99eb-8bf569c97 namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: 6247bc2c-4a54-4aaa-9684-e5fb4c84c685 kind: Event lastTimestamp: "2026-08-01T00:09:02Z" message: 'Created pod: sequence-graph-d99eb-8bf569c97-62v27' metadata: creationTimestamp: "2026-08-01T00:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:09:02Z" name: sequence-graph-d99eb-8bf569c97.18c785d896dbaf55 namespace: kserve-ci-e2e-test resourceVersion: "21974" uid: 18b20360-cde7-455e-9305-b990eb944622 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:09:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-d99eb namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: e3578268-ef37-4c17-b61b-6a60b7c79758 kind: Event lastTimestamp: "2026-08-01T00:09:02Z" message: Scaled up replica set sequence-graph-d99eb-8bf569c97 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:09:02Z" name: sequence-graph-d99eb.18c785d896136ed9 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: 582ca24f-b154-4c19-8e0e-5eb84563d205 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:09:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-d99eb namespace: kserve-ci-e2e-test resourceVersion: "21983" uid: 6be010b9-f387-4914-a021-4fb53173effe kind: Event lastTimestamp: "2026-08-01T00:09:02Z" message: InferenceGraph [sequence-graph-d99eb] is Ready metadata: creationTimestamp: "2026-08-01T00:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:09:02Z" name: sequence-graph-d99eb.18c785d899901dda namespace: kserve-ci-e2e-test resourceVersion: "21985" uid: 07a110b3-d71f-43a6-b8e3-bf8a2245ecff reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:09:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d99eb namespace: kserve-ci-e2e-test resourceVersion: "21968" uid: 40f47807-a552-4d26-96c7-6b0f3333956e kind: Event lastTimestamp: "2026-08-01T00:09:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:09:17Z" name: sequence-graph-d99eb.18c785dc147cc2d2 namespace: kserve-ci-e2e-test resourceVersion: "22103" uid: 9b862967-ba13-44af-9c03-4abfd8be5667 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:09:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d99eb namespace: kserve-ci-e2e-test resourceVersion: "21968" uid: 40f47807-a552-4d26-96c7-6b0f3333956e kind: Event lastTimestamp: "2026-08-01T00:09:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:09:17Z" name: sequence-graph-d99eb.18c785dc14c0d5e6 namespace: kserve-ci-e2e-test resourceVersion: "22105" uid: 6a985512-c608-4643-9cd6-c35d4f8c0bf0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:09:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d99eb namespace: kserve-ci-e2e-test resourceVersion: "22104" uid: 40f47807-a552-4d26-96c7-6b0f3333956e kind: Event lastTimestamp: "2026-08-01T00:09:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:09:32Z" name: sequence-graph-d99eb.18c785df936254db namespace: kserve-ci-e2e-test resourceVersion: "22213" uid: be612c96-0ce1-4699-94f6-4d74bfefacf9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:09:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d99eb namespace: kserve-ci-e2e-test resourceVersion: "22104" uid: 40f47807-a552-4d26-96c7-6b0f3333956e kind: Event lastTimestamp: "2026-08-01T00:09:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:09:32Z" name: sequence-graph-d99eb.18c785df93c0ec6a namespace: kserve-ci-e2e-test resourceVersion: "22215" uid: f44c96a6-7f0a-48c6-962b-b1c4b81755ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:07:52.229403Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-df362-67945678d8-nr7bj namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: 65018b1f-2bdb-4b63-846c-99cdcc75aef5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-df362-67945678d8-nr7bj to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:07:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:07:52Z" name: sequence-graph-df362-67945678d8-nr7bj.18c785c83676ef1b namespace: kserve-ci-e2e-test resourceVersion: "21175" uid: 8def8372-a77e-46b0-89b6-bb6e82ae51d3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:07:52Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-df362-67945678d8-nr7bj namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 65018b1f-2bdb-4b63-846c-99cdcc75aef5 kind: Event lastTimestamp: "2026-08-01T00:07:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-df362-serving-cert" not found' metadata: creationTimestamp: "2026-08-01T00:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:52Z" name: sequence-graph-df362-67945678d8-nr7bj.18c785c84248595d namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: 58525978-de14-484b-b79e-894d6e64cca2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:54Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-df362-67945678d8-nr7bj namespace: kserve-ci-e2e-test resourceVersion: "21181" uid: 65018b1f-2bdb-4b63-846c-99cdcc75aef5 kind: Event lastTimestamp: "2026-08-01T00:07:54Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:07:54Z" name: sequence-graph-df362-67945678d8-nr7bj.18c785c8a98074b0 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: a3ec9a72-59db-4aaa-8a1f-a41deab61aec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-df362} kind: Pod name: sequence-graph-df362-67945678d8-nr7bj namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 65018b1f-2bdb-4b63-846c-99cdcc75aef5 kind: Event lastTimestamp: "2026-08-01T00:07:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:56dbaa30789264fbe80634bdc08e7f43bcfc19e5006180ec98cfd4cb6b8d99d6" already present on machine metadata: creationTimestamp: "2026-08-01T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:54Z" name: sequence-graph-df362-67945678d8-nr7bj.18c785c8aabf4913 namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: abdaa21d-67b9-4e23-ac07-81d1dad83953 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-df362} kind: Pod name: sequence-graph-df362-67945678d8-nr7bj namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 65018b1f-2bdb-4b63-846c-99cdcc75aef5 kind: Event lastTimestamp: "2026-08-01T00:07:54Z" message: 'Created container: sequence-graph-df362' metadata: creationTimestamp: "2026-08-01T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:54Z" name: sequence-graph-df362-67945678d8-nr7bj.18c785c8adc169bc namespace: kserve-ci-e2e-test resourceVersion: "21208" uid: 42ca0745-043b-4af0-9fa0-855594d7f88d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-df362} kind: Pod name: sequence-graph-df362-67945678d8-nr7bj namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 65018b1f-2bdb-4b63-846c-99cdcc75aef5 kind: Event lastTimestamp: "2026-08-01T00:07:54Z" message: Started container sequence-graph-df362 metadata: creationTimestamp: "2026-08-01T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:54Z" name: sequence-graph-df362-67945678d8-nr7bj.18c785c8ae38720a namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: 76a8041b-dae0-4f8a-9dcb-dbc9058f8910 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-df362} kind: Pod name: sequence-graph-df362-67945678d8-nr7bj namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 65018b1f-2bdb-4b63-846c-99cdcc75aef5 kind: Event lastTimestamp: "2026-08-01T00:08:02Z" message: Stopping container sequence-graph-df362 metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:02Z" name: sequence-graph-df362-67945678d8-nr7bj.18c785ca8fb6e04d namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: fba7caa7-123a-47b0-b630-c72c5cb4b317 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:08:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-df362} kind: Pod name: sequence-graph-df362-67945678d8-nr7bj namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 65018b1f-2bdb-4b63-846c-99cdcc75aef5 kind: Event lastTimestamp: "2026-08-01T00:08:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-01T00:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:30Z" name: sequence-graph-df362-67945678d8-nr7bj.18c785cb51299803 namespace: kserve-ci-e2e-test resourceVersion: "21699" uid: 958b3493-948b-4998-b690-6947904ef657 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-df362-67945678d8 namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 4938f540-b941-4ccf-a008-17a2a6722aa6 kind: Event lastTimestamp: "2026-08-01T00:07:52Z" message: 'Created pod: sequence-graph-df362-67945678d8-nr7bj' metadata: creationTimestamp: "2026-08-01T00:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:52Z" name: sequence-graph-df362-67945678d8.18c785c836096b48 namespace: kserve-ci-e2e-test resourceVersion: "21171" uid: aff217c0-fae7-4a10-8326-7d1389abcd12 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-df362 namespace: kserve-ci-e2e-test resourceVersion: "21164" uid: a87cb794-0aa4-4b31-87ee-e6dbaf7158f3 kind: Event lastTimestamp: "2026-08-01T00:07:52Z" message: Scaled up replica set sequence-graph-df362-67945678d8 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:52Z" name: sequence-graph-df362.18c785c8352b2e40 namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: 4466213e-6359-4a71-9804-d5be768e0603 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-df362 namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: d9cdc352-50b8-4481-a1f3-e7e7b001e6ba kind: Event lastTimestamp: "2026-08-01T00:07:52Z" message: InferenceGraph [sequence-graph-df362] is Ready metadata: creationTimestamp: "2026-08-01T00:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:07:52Z" name: sequence-graph-df362.18c785c83887b242 namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: ddce4a46-d61e-42eb-b2ae-e3856e9ce95c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:18:38.093456Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-2f013-5f984b6955-7x5nv namespace: kserve-ci-e2e-test resourceVersion: "26926" uid: 3061cbb5-a5f7-4911-8b37-d33a4dc48f7b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-2f013-5f984b6955-7x5nv to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:18:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:18:38Z" name: splitter-graph-2f013-5f984b6955-7x5nv.18c7865e96f65499 namespace: kserve-ci-e2e-test resourceVersion: "26931" uid: 09df91d5-0163-43c0-94b4-d3410a2d7187 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:38Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-2f013-5f984b6955-7x5nv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 3061cbb5-a5f7-4911-8b37-d33a4dc48f7b kind: Event lastTimestamp: "2026-08-01T00:18:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-2f013-serving-cert" not found' metadata: creationTimestamp: "2026-08-01T00:18:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:18:38Z" name: splitter-graph-2f013-5f984b6955-7x5nv.18c7865ea0d8ee56 namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 1601b437-bb39-45a0-8327-546358fc1ef5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:39Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-2f013-5f984b6955-7x5nv namespace: kserve-ci-e2e-test resourceVersion: "26935" uid: 3061cbb5-a5f7-4911-8b37-d33a4dc48f7b kind: Event lastTimestamp: "2026-08-01T00:18:39Z" message: Add eth0 [10.133.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:18:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:18:39Z" name: splitter-graph-2f013-5f984b6955-7x5nv.18c7865ed45f76bf namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 9fa0164b-2531-4ed7-be4d-7b0953ad6ca5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2f013} kind: Pod name: splitter-graph-2f013-5f984b6955-7x5nv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 3061cbb5-a5f7-4911-8b37-d33a4dc48f7b kind: Event lastTimestamp: "2026-08-01T00:18:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:56dbaa30789264fbe80634bdc08e7f43bcfc19e5006180ec98cfd4cb6b8d99d6" already present on machine metadata: creationTimestamp: "2026-08-01T00:18:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:18:39Z" name: splitter-graph-2f013-5f984b6955-7x5nv.18c7865ed59d462a namespace: kserve-ci-e2e-test resourceVersion: "26954" uid: 35a20087-f166-421d-ab4d-b4d7d046a033 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2f013} kind: Pod name: splitter-graph-2f013-5f984b6955-7x5nv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 3061cbb5-a5f7-4911-8b37-d33a4dc48f7b kind: Event lastTimestamp: "2026-08-01T00:18:39Z" message: 'Created container: splitter-graph-2f013' metadata: creationTimestamp: "2026-08-01T00:18:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:18:39Z" name: splitter-graph-2f013-5f984b6955-7x5nv.18c7865ed8ada23d namespace: kserve-ci-e2e-test resourceVersion: "26956" uid: eee4dc7c-b462-4a00-8d05-a13ee14708fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2f013} kind: Pod name: splitter-graph-2f013-5f984b6955-7x5nv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 3061cbb5-a5f7-4911-8b37-d33a4dc48f7b kind: Event lastTimestamp: "2026-08-01T00:18:39Z" message: Started container splitter-graph-2f013 metadata: creationTimestamp: "2026-08-01T00:18:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:18:39Z" name: splitter-graph-2f013-5f984b6955-7x5nv.18c7865ed9271338 namespace: kserve-ci-e2e-test resourceVersion: "26957" uid: 8cd358f9-8378-4d8a-8eb4-39f55bf43ece reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:26:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2f013} kind: Pod name: splitter-graph-2f013-5f984b6955-7x5nv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 3061cbb5-a5f7-4911-8b37-d33a4dc48f7b kind: Event lastTimestamp: "2026-08-01T00:26:52Z" message: Stopping container splitter-graph-2f013 metadata: creationTimestamp: "2026-08-01T00:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:26:52Z" name: splitter-graph-2f013-5f984b6955-7x5nv.18c786d1c437b70b namespace: kserve-ci-e2e-test resourceVersion: "30372" uid: 26a093e6-3f8e-4d3c-972d-284f314c99be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2f013} kind: Pod name: splitter-graph-2f013-5f984b6955-7x5nv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 3061cbb5-a5f7-4911-8b37-d33a4dc48f7b kind: Event lastTimestamp: "2026-08-01T00:27:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-01T00:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:27:20Z" name: splitter-graph-2f013-5f984b6955-7x5nv.18c786d26e808438 namespace: kserve-ci-e2e-test resourceVersion: "30608" uid: 1b0c31e7-3f44-433d-93b7-a25ed77bf133 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-2f013-5f984b6955 namespace: kserve-ci-e2e-test resourceVersion: "26923" uid: 5dfd2998-6ce2-47c6-bb82-6309b4481f50 kind: Event lastTimestamp: "2026-08-01T00:18:38Z" message: 'Created pod: splitter-graph-2f013-5f984b6955-7x5nv' metadata: creationTimestamp: "2026-08-01T00:18:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:38Z" name: splitter-graph-2f013-5f984b6955.18c7865e969451d4 namespace: kserve-ci-e2e-test resourceVersion: "26927" uid: 39d9d5d9-9f31-4925-904d-42505b6af8d5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-2f013 namespace: kserve-ci-e2e-test resourceVersion: "26921" uid: 3e295f11-2372-4d73-a6b6-0ed2e66f9500 kind: Event lastTimestamp: "2026-08-01T00:18:38Z" message: Scaled up replica set splitter-graph-2f013-5f984b6955 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:18:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:38Z" name: splitter-graph-2f013.18c7865e95dfcf60 namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: e4010de2-db98-4dd6-bb03-0e73c4c5f060 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-2f013 namespace: kserve-ci-e2e-test resourceVersion: "26937" uid: 379e2d47-da7e-489c-bd1f-e397babbe303 kind: Event lastTimestamp: "2026-08-01T00:18:38Z" message: InferenceGraph [splitter-graph-2f013] is Ready metadata: creationTimestamp: "2026-08-01T00:18:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:18:38Z" name: splitter-graph-2f013.18c7865e99b57bf1 namespace: kserve-ci-e2e-test resourceVersion: "26939" uid: cf665920-8dbc-4d46-a885-4c7a7ff19864 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-2f013 namespace: kserve-ci-e2e-test resourceVersion: "26922" uid: 22fc9049-2d4e-4e6f-bdd4-d75356fe807c kind: Event lastTimestamp: "2026-08-01T00:18:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:53Z" name: splitter-graph-2f013.18c78662147ec4e2 namespace: kserve-ci-e2e-test resourceVersion: "27062" uid: e503c00f-6958-49ee-b488-684bd35e1f80 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-2f013 namespace: kserve-ci-e2e-test resourceVersion: "26922" uid: 22fc9049-2d4e-4e6f-bdd4-d75356fe807c kind: Event lastTimestamp: "2026-08-01T00:18:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:53Z" name: splitter-graph-2f013.18c7866214df675c namespace: kserve-ci-e2e-test resourceVersion: "27063" uid: 8039ecf1-e633-46f8-afc8-6cb0c1d5209f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:19:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-2f013 namespace: kserve-ci-e2e-test resourceVersion: "27061" uid: 22fc9049-2d4e-4e6f-bdd4-d75356fe807c kind: Event lastTimestamp: "2026-08-01T00:19:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:19:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:19:08Z" name: splitter-graph-2f013.18c78665938195c3 namespace: kserve-ci-e2e-test resourceVersion: "27169" uid: 72a51cde-a20a-4f33-b655-770f0f73200b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:19:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-2f013 namespace: kserve-ci-e2e-test resourceVersion: "27061" uid: 22fc9049-2d4e-4e6f-bdd4-d75356fe807c kind: Event lastTimestamp: "2026-08-01T00:19:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:19:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:19:08Z" name: splitter-graph-2f013.18c7866593c410c8 namespace: kserve-ci-e2e-test resourceVersion: "27171" uid: 10423cf8-cbd1-45e3-a2ab-223da89bfd24 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:17:27.740888Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-d0074-8f77cc87-zwsk9 namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: a8a3c57c-c5b7-4e0b-9c1f-61c8be085c40 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-d0074-8f77cc87-zwsk9 to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:17:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:17:27Z" name: splitter-graph-d0074-8f77cc87-zwsk9.18c7864e359f53ae namespace: kserve-ci-e2e-test resourceVersion: "26100" uid: 06d8987c-d87b-46b6-84e4-2019e7bf38dc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:28Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-d0074-8f77cc87-zwsk9 namespace: kserve-ci-e2e-test resourceVersion: "26103" uid: a8a3c57c-c5b7-4e0b-9c1f-61c8be085c40 kind: Event lastTimestamp: "2026-08-01T00:17:28Z" message: Add eth0 [10.133.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:17:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:17:28Z" name: splitter-graph-d0074-8f77cc87-zwsk9.18c7864e4f7c53e6 namespace: kserve-ci-e2e-test resourceVersion: "26116" uid: 4eb6ede5-e2e2-4e57-abeb-9bd009c19047 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d0074} kind: Pod name: splitter-graph-d0074-8f77cc87-zwsk9 namespace: kserve-ci-e2e-test resourceVersion: "26096" uid: a8a3c57c-c5b7-4e0b-9c1f-61c8be085c40 kind: Event lastTimestamp: "2026-08-01T00:17:28Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:56dbaa30789264fbe80634bdc08e7f43bcfc19e5006180ec98cfd4cb6b8d99d6" already present on machine metadata: creationTimestamp: "2026-08-01T00:17:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:28Z" name: splitter-graph-d0074-8f77cc87-zwsk9.18c7864e50b92925 namespace: kserve-ci-e2e-test resourceVersion: "26118" uid: b237df94-251c-4dcc-b41c-093a25aecf01 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d0074} kind: Pod name: splitter-graph-d0074-8f77cc87-zwsk9 namespace: kserve-ci-e2e-test resourceVersion: "26096" uid: a8a3c57c-c5b7-4e0b-9c1f-61c8be085c40 kind: Event lastTimestamp: "2026-08-01T00:17:28Z" message: 'Created container: splitter-graph-d0074' metadata: creationTimestamp: "2026-08-01T00:17:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:28Z" name: splitter-graph-d0074-8f77cc87-zwsk9.18c7864e548fb96f namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: a35ea58e-fefd-4691-9465-67333cad85cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d0074} kind: Pod name: splitter-graph-d0074-8f77cc87-zwsk9 namespace: kserve-ci-e2e-test resourceVersion: "26096" uid: a8a3c57c-c5b7-4e0b-9c1f-61c8be085c40 kind: Event lastTimestamp: "2026-08-01T00:17:28Z" message: Started container splitter-graph-d0074 metadata: creationTimestamp: "2026-08-01T00:17:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:28Z" name: splitter-graph-d0074-8f77cc87-zwsk9.18c7864e55054469 namespace: kserve-ci-e2e-test resourceVersion: "26121" uid: 2f726109-86c1-4a34-8339-190e06902049 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d0074} kind: Pod name: splitter-graph-d0074-8f77cc87-zwsk9 namespace: kserve-ci-e2e-test resourceVersion: "26096" uid: a8a3c57c-c5b7-4e0b-9c1f-61c8be085c40 kind: Event lastTimestamp: "2026-08-01T00:17:37Z" message: Stopping container splitter-graph-d0074 metadata: creationTimestamp: "2026-08-01T00:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:37Z" name: splitter-graph-d0074-8f77cc87-zwsk9.18c786508f58eb5e namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 152932da-d61a-4113-83e1-a719b66abefa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:17:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d0074} kind: Pod name: splitter-graph-d0074-8f77cc87-zwsk9 namespace: kserve-ci-e2e-test resourceVersion: "26096" uid: a8a3c57c-c5b7-4e0b-9c1f-61c8be085c40 kind: Event lastTimestamp: "2026-08-01T00:18:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-01T00:17:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:18:05Z" name: splitter-graph-d0074-8f77cc87-zwsk9.18c786512068974f namespace: kserve-ci-e2e-test resourceVersion: "26579" uid: aebd4ab3-056a-401f-a52a-1cf3caf3760d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-d0074-8f77cc87 namespace: kserve-ci-e2e-test resourceVersion: "26090" uid: 7dad1d5d-ff7c-484e-8b7e-6ff87a4cda07 kind: Event lastTimestamp: "2026-08-01T00:17:27Z" message: 'Created pod: splitter-graph-d0074-8f77cc87-zwsk9' metadata: creationTimestamp: "2026-08-01T00:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:27Z" name: splitter-graph-d0074-8f77cc87.18c7864e35390f53 namespace: kserve-ci-e2e-test resourceVersion: "26095" uid: 881d32d5-9181-43dc-9c4f-8c6b0eb3a2dc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-d0074 namespace: kserve-ci-e2e-test resourceVersion: "26088" uid: 08485f07-aaf5-465b-9a95-edd924821706 kind: Event lastTimestamp: "2026-08-01T00:17:27Z" message: Scaled up replica set splitter-graph-d0074-8f77cc87 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:27Z" name: splitter-graph-d0074.18c7864e348d2104 namespace: kserve-ci-e2e-test resourceVersion: "26092" uid: a2af56e1-0af9-4ec4-b5e3-883695cfaeef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-d0074 namespace: kserve-ci-e2e-test resourceVersion: "26105" uid: fd500704-e105-40d1-99c9-e799d2908317 kind: Event lastTimestamp: "2026-08-01T00:17:27Z" message: InferenceGraph [splitter-graph-d0074] is Ready metadata: creationTimestamp: "2026-08-01T00:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:17:27Z" name: splitter-graph-d0074.18c7864e3808074d namespace: kserve-ci-e2e-test resourceVersion: "26109" uid: 7ed2791b-40cd-488d-a569-eab2e73b3414 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:07:12.663370Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-01c7a-predictor-774957f645-hvd2w namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: b6eed2b1-cef9-457a-b42d-99effb74a9a4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-01c7a-predictor-774957f645-hvd2w to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:07:12Z" name: success-200-isvc-01c7a-predictor-774957f645-hvd2w.18c785bf00253b2f namespace: kserve-ci-e2e-test resourceVersion: "20740" uid: 71a0d9f6-d2ed-4fb4-8555-92867ef7b056 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-01c7a-predictor-774957f645-hvd2w namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: b6eed2b1-cef9-457a-b42d-99effb74a9a4 kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-01c7a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:12Z" name: success-200-isvc-01c7a-predictor-774957f645-hvd2w.18c785bf08117d22 namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 71621911-530e-4d2b-8ac4-df674ad5079d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-01c7a-predictor-774957f645-hvd2w namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: b6eed2b1-cef9-457a-b42d-99effb74a9a4 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:07:13Z" name: success-200-isvc-01c7a-predictor-774957f645-hvd2w.18c785bf3dd474e2 namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: 31033dcb-7938-4622-a952-2bce69b9c139 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-01c7a-predictor-774957f645-hvd2w namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: b6eed2b1-cef9-457a-b42d-99effb74a9a4 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:13Z" name: success-200-isvc-01c7a-predictor-774957f645-hvd2w.18c785bf3f20be5a namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: b17dbea0-0ca7-4a95-a0b8-56b0ab3926d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-01c7a-predictor-774957f645-hvd2w namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: b6eed2b1-cef9-457a-b42d-99effb74a9a4 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:13Z" name: success-200-isvc-01c7a-predictor-774957f645-hvd2w.18c785bf4254c5db namespace: kserve-ci-e2e-test resourceVersion: "20793" uid: 3bdcb7c7-7c47-41c1-90cd-93c59d472715 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-01c7a-predictor-774957f645-hvd2w namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: b6eed2b1-cef9-457a-b42d-99effb74a9a4 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:13Z" name: success-200-isvc-01c7a-predictor-774957f645-hvd2w.18c785bf42cba024 namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: bf6ca379-4ff5-4d7a-b1e4-ce51e987bdaf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-01c7a-predictor-774957f645-hvd2w namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: b6eed2b1-cef9-457a-b42d-99effb74a9a4 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:13Z" name: success-200-isvc-01c7a-predictor-774957f645-hvd2w.18c785bf42d5af50 namespace: kserve-ci-e2e-test resourceVersion: "20795" uid: a7233cdf-c5de-4c94-b3d6-4e1bbc4c1997 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-01c7a-predictor-774957f645-hvd2w namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: b6eed2b1-cef9-457a-b42d-99effb74a9a4 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:13Z" name: success-200-isvc-01c7a-predictor-774957f645-hvd2w.18c785bf4625178e namespace: kserve-ci-e2e-test resourceVersion: "20800" uid: 581a1fa2-2ae4-4f9c-a110-f1a6cb3deb20 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-01c7a-predictor-774957f645-hvd2w namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: b6eed2b1-cef9-457a-b42d-99effb74a9a4 kind: Event lastTimestamp: "2026-08-01T00:07:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:13Z" name: success-200-isvc-01c7a-predictor-774957f645-hvd2w.18c785bf46a8726e namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: b5201ab9-9e76-447e-8641-d1b191258ccc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-08-01T00:07:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-01c7a-predictor-774957f645-hvd2w namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: b6eed2b1-cef9-457a-b42d-99effb74a9a4 kind: Event lastTimestamp: "2026-08-01T00:16:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:30Z" name: success-200-isvc-01c7a-predictor-774957f645-hvd2w.18c785bf64c394b0 namespace: kserve-ci-e2e-test resourceVersion: "25186" uid: 4171994d-89f4-4bec-90fb-af4b4fce4ace reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-01c7a-predictor-774957f645-hvd2w namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: b6eed2b1-cef9-457a-b42d-99effb74a9a4 kind: Event lastTimestamp: "2026-08-01T00:16:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:27Z" name: success-200-isvc-01c7a-predictor-774957f645-hvd2w.18c78640375a7525 namespace: kserve-ci-e2e-test resourceVersion: "25075" uid: f0afe8a5-1781-4903-83c0-f0d45ba510a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-01c7a-predictor-774957f645-hvd2w namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: b6eed2b1-cef9-457a-b42d-99effb74a9a4 kind: Event lastTimestamp: "2026-08-01T00:16:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:27Z" name: success-200-isvc-01c7a-predictor-774957f645-hvd2w.18c78640375b126f namespace: kserve-ci-e2e-test resourceVersion: "25076" uid: fdd605fb-3817-4bbc-8548-79cad616b3be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-01c7a-predictor-774957f645-hvd2w namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: b6eed2b1-cef9-457a-b42d-99effb74a9a4 kind: Event lastTimestamp: "2026-08-01T00:16:30Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:30Z" name: success-200-isvc-01c7a-predictor-774957f645-hvd2w.18c78640d94b68c8 namespace: kserve-ci-e2e-test resourceVersion: "25185" uid: 554c617b-2a04-4559-a683-ce9c767a64aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-01c7a-predictor-774957f645 namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: 1e418a53-d080-4a6b-8737-00c49b64f935 kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: 'Created pod: success-200-isvc-01c7a-predictor-774957f645-hvd2w' metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:12Z" name: success-200-isvc-01c7a-predictor-774957f645.18c785beffc06354 namespace: kserve-ci-e2e-test resourceVersion: "20736" uid: ae64d5d3-6c2c-4043-886e-09fc256aa086 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-01c7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: 00c2e1d5-9c92-4474-9b52-67d14772139b kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: Scaled up replica set success-200-isvc-01c7a-predictor-774957f645 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:12Z" name: success-200-isvc-01c7a-predictor.18c785befe2a9d3f namespace: kserve-ci-e2e-test resourceVersion: "20728" uid: dfccd8e4-6de0-4800-85e7-02a61c7ea752 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-01c7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: 63e135b8-ec20-4538-b1eb-7bc90a55b54e kind: Event lastTimestamp: "2026-08-01T00:07:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:27Z" name: success-200-isvc-01c7a-predictor.18c785c27d38c44a namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: d3d06d37-b0c1-419e-ae6e-a170f483ad63 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-01c7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: 63e135b8-ec20-4538-b1eb-7bc90a55b54e kind: Event lastTimestamp: "2026-08-01T00:07:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:27Z" name: success-200-isvc-01c7a-predictor.18c785c27ddceb3b namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: f9aa93b6-6451-4d64-8da2-2d05fd6b6c9a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-01T00:07:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-01c7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: 63e135b8-ec20-4538-b1eb-7bc90a55b54e kind: Event lastTimestamp: "2026-08-01T00:08:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:27Z" name: success-200-isvc-01c7a-predictor.18c785c5fcdadc4c namespace: kserve-ci-e2e-test resourceVersion: "21671" uid: 5f94e807-3f33-42e3-b6cc-f1d19295ef70 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-01T00:07:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-01c7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: 63e135b8-ec20-4538-b1eb-7bc90a55b54e kind: Event lastTimestamp: "2026-08-01T00:08:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:07:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:27Z" name: success-200-isvc-01c7a-predictor.18c785c5fd3e0299 namespace: kserve-ci-e2e-test resourceVersion: "21672" uid: adb8f903-d6c8-4c7a-aca8-7ab6772c6560 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-01c7a namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: c23bf515-625a-4b4c-bf83-4972490cb54e kind: Event lastTimestamp: "2026-08-01T00:08:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:08:00Z" name: success-200-isvc-01c7a.18c785befe420ca4 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 0921914d-781d-488c-9ec3-1ce6435e483c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-01c7a namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: c23bf515-625a-4b4c-bf83-4972490cb54e kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-01c7a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-01c7a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:07:12Z" name: success-200-isvc-01c7a.18c785befeba94b3 namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 3bf3700a-b701-42bb-9c0a-cec1227ab9a9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-01c7a namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: c23bf515-625a-4b4c-bf83-4972490cb54e kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-01c7a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:07:12Z" name: success-200-isvc-01c7a.18c785befebae983 namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: 31a72e5f-ecf6-4665-9636-731f0d1bb30d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-01c7a namespace: kserve-ci-e2e-test resourceVersion: "21264" uid: c23bf515-625a-4b4c-bf83-4972490cb54e kind: Event lastTimestamp: "2026-08-01T00:08:00Z" message: InferenceService [success-200-isvc-01c7a] is Ready metadata: creationTimestamp: "2026-08-01T00:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:08:00Z" name: success-200-isvc-01c7a.18c785ca20025ca0 namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: 524d456d-143f-43d3-b41b-1255bf8ef9a5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:17:38.027619Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2f013-predictor-bb647f765-5tkvs namespace: kserve-ci-e2e-test resourceVersion: "26266" uid: 915a3a14-b2de-4ab3-ac4a-5215dcecab1f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2f013-predictor-bb647f765-5tkvs to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:17:38Z" name: success-200-isvc-2f013-predictor-bb647f765-5tkvs.18c786509ac26d3e namespace: kserve-ci-e2e-test resourceVersion: "26271" uid: ab6cf259-b7e3-437e-aea2-9f1f321ceb98 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2f013-predictor-bb647f765-5tkvs namespace: kserve-ci-e2e-test resourceVersion: "26278" uid: 915a3a14-b2de-4ab3-ac4a-5215dcecab1f kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: Add eth0 [10.133.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:17:38Z" name: success-200-isvc-2f013-predictor-bb647f765-5tkvs.18c78650b4af4a74 namespace: kserve-ci-e2e-test resourceVersion: "26317" uid: bab8deb4-5e50-41c5-a53d-28c86c2e18bb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2f013-predictor-bb647f765-5tkvs namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 915a3a14-b2de-4ab3-ac4a-5215dcecab1f kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: success-200-isvc-2f013-predictor-bb647f765-5tkvs.18c78650b5f23b3b namespace: kserve-ci-e2e-test resourceVersion: "26319" uid: 8cf5c411-8b8c-4df4-b9aa-73bb4ea82ece reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2f013-predictor-bb647f765-5tkvs namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 915a3a14-b2de-4ab3-ac4a-5215dcecab1f kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: success-200-isvc-2f013-predictor-bb647f765-5tkvs.18c78650b8f7213e namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: c471c74c-cf1e-44dc-bb9e-985c6aecfbef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2f013-predictor-bb647f765-5tkvs namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 915a3a14-b2de-4ab3-ac4a-5215dcecab1f kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: success-200-isvc-2f013-predictor-bb647f765-5tkvs.18c78650b9712e9a namespace: kserve-ci-e2e-test resourceVersion: "26321" uid: 50789fd8-13f1-4c24-b25c-21c46afa23b4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2f013-predictor-bb647f765-5tkvs namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 915a3a14-b2de-4ab3-ac4a-5215dcecab1f kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: success-200-isvc-2f013-predictor-bb647f765-5tkvs.18c78650b97c779e namespace: kserve-ci-e2e-test resourceVersion: "26322" uid: f6419714-8372-4056-a780-859db3dced39 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2f013-predictor-bb647f765-5tkvs namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 915a3a14-b2de-4ab3-ac4a-5215dcecab1f kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: success-200-isvc-2f013-predictor-bb647f765-5tkvs.18c78650bc399471 namespace: kserve-ci-e2e-test resourceVersion: "26325" uid: 478b81df-c8e5-4723-9270-3269f90d9444 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2f013-predictor-bb647f765-5tkvs namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 915a3a14-b2de-4ab3-ac4a-5215dcecab1f kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:38Z" name: success-200-isvc-2f013-predictor-bb647f765-5tkvs.18c78650bcafe06c namespace: kserve-ci-e2e-test resourceVersion: "26327" uid: 556d763d-a8c3-471b-a4fa-e8a08a7490f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:17:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2f013-predictor-bb647f765-5tkvs namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 915a3a14-b2de-4ab3-ac4a-5215dcecab1f kind: Event lastTimestamp: "2026-08-01T00:18:16Z" message: 'Readiness probe failed: dial tcp 10.133.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:17:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:18:16Z" name: success-200-isvc-2f013-predictor-bb647f765-5tkvs.18c7865123b672be namespace: kserve-ci-e2e-test resourceVersion: "26688" uid: 62443b78-83e9-4f72-97ff-3b3a327d7cbd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:26:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2f013-predictor-bb647f765-5tkvs namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 915a3a14-b2de-4ab3-ac4a-5215dcecab1f kind: Event lastTimestamp: "2026-08-01T00:26:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:26:52Z" name: success-200-isvc-2f013-predictor-bb647f765-5tkvs.18c786d1cbe00dec namespace: kserve-ci-e2e-test resourceVersion: "30389" uid: a08d1741-3a8f-49bf-872a-7dbc89f1a6cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:26:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2f013-predictor-bb647f765-5tkvs namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 915a3a14-b2de-4ab3-ac4a-5215dcecab1f kind: Event lastTimestamp: "2026-08-01T00:26:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:26:52Z" name: success-200-isvc-2f013-predictor-bb647f765-5tkvs.18c786d1cbe0b3a8 namespace: kserve-ci-e2e-test resourceVersion: "30390" uid: 99fc348b-2fd5-46ac-a48c-254a356c1e4f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2f013-predictor-bb647f765 namespace: kserve-ci-e2e-test resourceVersion: "26252" uid: 749257b7-722c-4a30-91d0-92b85157da50 kind: Event lastTimestamp: "2026-08-01T00:17:38Z" message: 'Created pod: success-200-isvc-2f013-predictor-bb647f765-5tkvs' metadata: creationTimestamp: "2026-08-01T00:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:38Z" name: success-200-isvc-2f013-predictor-bb647f765.18c786509a542f2c namespace: kserve-ci-e2e-test resourceVersion: "26267" uid: 31f188a2-60b1-41a2-a78c-0875e5ee4197 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2f013-predictor namespace: kserve-ci-e2e-test resourceVersion: "26247" uid: 69756d98-479b-4c7b-9488-efea4160c72c kind: Event lastTimestamp: "2026-08-01T00:17:37Z" message: Scaled up replica set success-200-isvc-2f013-predictor-bb647f765 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:37Z" name: success-200-isvc-2f013-predictor.18c786509887d9f7 namespace: kserve-ci-e2e-test resourceVersion: "26255" uid: 83da82c7-a294-4a39-8839-1f01b191e118 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:17:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2f013-predictor namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 0bf45e47-a9b1-4400-a968-acade37a1c1b kind: Event lastTimestamp: "2026-08-01T00:18:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:23Z" name: success-200-isvc-2f013-predictor.18c7865416e08043 namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: c349c7da-68e2-40e0-9dc2-f031f000b37b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:17:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2f013-predictor namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 0bf45e47-a9b1-4400-a968-acade37a1c1b kind: Event lastTimestamp: "2026-08-01T00:18:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:23Z" name: success-200-isvc-2f013-predictor.18c78654173fa657 namespace: kserve-ci-e2e-test resourceVersion: "26775" uid: f0f393ef-a16f-498a-8188-7abc84bf7344 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:18:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2f013-predictor namespace: kserve-ci-e2e-test resourceVersion: "26473" uid: 0bf45e47-a9b1-4400-a968-acade37a1c1b kind: Event lastTimestamp: "2026-08-01T00:18:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:18:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:53Z" name: success-200-isvc-2f013-predictor.18c7865e935a89ea namespace: kserve-ci-e2e-test resourceVersion: "27058" uid: ed3ad57a-b035-4841-96ea-25d0762478c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:18:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2f013-predictor namespace: kserve-ci-e2e-test resourceVersion: "26473" uid: 0bf45e47-a9b1-4400-a968-acade37a1c1b kind: Event lastTimestamp: "2026-08-01T00:18:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:18:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:53Z" name: success-200-isvc-2f013-predictor.18c7865e93b98f52 namespace: kserve-ci-e2e-test resourceVersion: "27059" uid: 2db5d318-7252-4ec1-9a4a-9cb20ad9155c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-01T00:17:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2f013 namespace: kserve-ci-e2e-test resourceVersion: "26226" uid: 545a0345-9db4-4697-abb3-6d7fe1a5b5e7 kind: Event lastTimestamp: "2026-08-01T00:18:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:18:26Z" name: success-200-isvc-2f013.18c7865098a52260 namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 0bd492bc-95c9-44e8-be09-50c76fb31955 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2f013 namespace: kserve-ci-e2e-test resourceVersion: "26821" uid: 545a0345-9db4-4697-abb3-6d7fe1a5b5e7 kind: Event lastTimestamp: "2026-08-01T00:18:26Z" message: InferenceService [success-200-isvc-2f013] is Ready metadata: creationTimestamp: "2026-08-01T00:18:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:18:26Z" name: success-200-isvc-2f013.18c7865bdea2205d namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: ad8e16c7-203d-4f1c-80e8-d2824dcda2be reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T23:56:47.229669Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: fbac4361-0369-4cea-ab12-03a4e4ea1171 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-32541-predictor-7ff4c79d55-9mbgd to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T23:56:47Z" name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd.18c7852d616450ca namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: 55ba9c9b-139d-47b6-9bd0-990eaebadbdd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: fbac4361-0369-4cea-ab12-03a4e4ea1171 kind: Event lastTimestamp: "2026-07-31T23:56:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-32541-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:47Z" name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd.18c7852d686119cc namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 04f58184-1b3b-4fdc-884b-852397d655e6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: fbac4361-0369-4cea-ab12-03a4e4ea1171 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T23:56:48Z" name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd.18c7852d9eecb566 namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: 19554118-ae60-4548-a451-6cb317c9886e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: fbac4361-0369-4cea-ab12-03a4e4ea1171 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:48Z" name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd.18c7852da03872e7 namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: ca9b93d2-3862-4225-8271-e62a08fe9fa0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: fbac4361-0369-4cea-ab12-03a4e4ea1171 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:48Z" name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd.18c7852da392eadb namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: 9c5d71ad-586b-4651-9d60-7c63511881bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: fbac4361-0369-4cea-ab12-03a4e4ea1171 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:48Z" name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd.18c7852da409eb77 namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: e4f79716-f1c9-48d6-906f-908de4898d88 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: fbac4361-0369-4cea-ab12-03a4e4ea1171 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:48Z" name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd.18c7852da4158998 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 87bddb4c-86af-411b-a144-0dfcadddf27e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: fbac4361-0369-4cea-ab12-03a4e4ea1171 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:48Z" name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd.18c7852da76d93d5 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: 557315af-8ce3-4f15-9d19-47cc3d032263 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: fbac4361-0369-4cea-ab12-03a4e4ea1171 kind: Event lastTimestamp: "2026-07-31T23:56:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:48Z" name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd.18c7852da7e33279 namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: e6aa3147-fcf9-466d-a39a-ba8bc8356134 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T23:56:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: fbac4361-0369-4cea-ab12-03a4e4ea1171 kind: Event lastTimestamp: "2026-07-31T23:57:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:26Z" name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd.18c7852e1f4d71ed namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: b96afb00-b89f-441a-8e01-7f38395067df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: fbac4361-0369-4cea-ab12-03a4e4ea1171 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd.18c785ae9a73e753 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: b821f3d9-90b1-4ded-bdad-281908eded7a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: fbac4361-0369-4cea-ab12-03a4e4ea1171 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd.18c785ae9a74874b namespace: kserve-ci-e2e-test resourceVersion: "19826" uid: 76536635-1780-4c3d-a1d8-5dec753a0b4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-32541-predictor-7ff4c79d55 namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: a7d7c0ec-1f5e-409b-90c0-06dcb2f1241b kind: Event lastTimestamp: "2026-07-31T23:56:47Z" message: 'Created pod: success-200-isvc-32541-predictor-7ff4c79d55-9mbgd' metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:47Z" name: success-200-isvc-32541-predictor-7ff4c79d55.18c7852d60e53403 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 711c8e49-c36b-4d4f-aabb-f87fec90e869 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-32541-predictor namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: ebfbde25-f4da-4112-92c3-0e9e3fe5794f kind: Event lastTimestamp: "2026-07-31T23:56:47Z" message: Scaled up replica set success-200-isvc-32541-predictor-7ff4c79d55 from 0 to 1 metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:47Z" name: success-200-isvc-32541-predictor.18c7852d5ec3cf8c namespace: kserve-ci-e2e-test resourceVersion: "15407" uid: 132b29f2-f33f-4e42-99d1-9539859514b1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T23:57:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32541-predictor namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 4a3337c1-13c0-40de-a9d9-b1bdf5c8cc8b kind: Event lastTimestamp: "2026-07-31T23:57:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:32Z" name: success-200-isvc-32541-predictor.18c78530dd2c7891 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: 6f41fb27-c691-4eaa-99c0-495d25102122 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T23:57:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32541-predictor namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 4a3337c1-13c0-40de-a9d9-b1bdf5c8cc8b kind: Event lastTimestamp: "2026-07-31T23:57:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:32Z" name: success-200-isvc-32541-predictor.18c78530dd8aed05 namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: d1968586-f4d8-4d5f-bee6-9edc7709ea59 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T23:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32541-predictor namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 4a3337c1-13c0-40de-a9d9-b1bdf5c8cc8b kind: Event lastTimestamp: "2026-07-31T23:58:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:17Z" name: success-200-isvc-32541-predictor.18c785345c30a551 namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 139eb1b4-7d6f-4ec6-8c23-49552131eebd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T23:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32541-predictor namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 4a3337c1-13c0-40de-a9d9-b1bdf5c8cc8b kind: Event lastTimestamp: "2026-07-31T23:58:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:17Z" name: success-200-isvc-32541-predictor.18c785345c95660c namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: 4f348df4-91c9-47d8-9963-8dac81e533fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32541 namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: b4a926ec-6a16-43fa-8953-ca143aacad71 kind: Event lastTimestamp: "2026-07-31T23:57:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:36Z" name: success-200-isvc-32541.18c7852d5ec78211 namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: 06551123-60fa-4316-afb9-3a692774275e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32541 namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: b4a926ec-6a16-43fa-8953-ca143aacad71 kind: Event lastTimestamp: "2026-07-31T23:57:36Z" message: InferenceService [success-200-isvc-32541] is Ready metadata: creationTimestamp: "2026-07-31T23:57:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:36Z" name: success-200-isvc-32541.18c78538da6d92b8 namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: 64d24292-15ce-40bc-b03b-769fba1e80ff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T23:55:17.149195Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T23:55:17Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c78518682d61cc namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: 912b7ca3-0159-433f-9675-7253e575a245 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14397" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T23:55:17Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c7851882a58507 namespace: kserve-ci-e2e-test resourceVersion: "14490" uid: a4fd4a89-cb2a-4f08-990e-90f0a743af02 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1841" metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:17Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c785188410a800 namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 8f702d94-f3ca-45db-ba87-b2ca059f1a14 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: "2026-07-31T23:55:29Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1841" in 11.852s (11.852s including waiting). Image size: 1334931570 bytes.' metadata: creationTimestamp: "2026-07-31T23:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:29Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c7851b468b724f namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: 6012135a-8037-4f9c-bbb8-09ed22b76478 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: "2026-07-31T23:55:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T23:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:29Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c7851b4984720f namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 8adb5011-d3de-4d2e-9fa8-04877b33c785 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: "2026-07-31T23:55:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T23:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:29Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c7851b49fd5aee namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 544d08ed-ac78-4877-8738-4c7d16bf5e74 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: "2026-07-31T23:55:29Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-31T23:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:29Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c7851b4a0e43b7 namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: e54d4312-f13d-40a5-a38a-db53ae403cd6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: "2026-07-31T23:55:31Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.297s (2.297s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-31T23:55:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:31Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c7851bd2f9d450 namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: 26b24b87-db38-47f9-a94b-ccda31838c7a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: "2026-07-31T23:55:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T23:55:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:31Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c7851bd62e07f5 namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: 89cd0831-b699-4c70-bdf0-04852f2f5e2f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: "2026-07-31T23:55:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:55:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:55:31Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c7851bd6a429bd namespace: kserve-ci-e2e-test resourceVersion: "14648" uid: ec9765b6-0440-4cb5-a2ae-a84880f30da8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T23:55:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: "2026-07-31T23:56:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:49Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c7851c1beb9e12 namespace: kserve-ci-e2e-test resourceVersion: "15508" uid: 9a9901ad-0b61-4246-afc2-8ecb7bfb51e6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: "2026-07-31T23:56:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:47Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c7852d5d01fed5 namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 7a5bb588-d130-4b59-b336-5e0630dc84ad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: "2026-07-31T23:56:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:47Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c7852d5d03039f namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: 4a7753d0-832c-4184-b3c5-a947c0e349cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 3d9b874f-5a27-498b-93bc-d0771624ffd9 kind: Event lastTimestamp: "2026-07-31T23:56:49Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:49Z" name: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2.18c7852dce11ba6e namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: c9bf3e79-08f0-4c6d-89f9-597dda5d2975 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-716dc-predictor-6b5b687b9f namespace: kserve-ci-e2e-test resourceVersion: "14380" uid: 8312be13-f598-457e-9de2-74a928cd6703 kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: 'Created pod: success-200-isvc-716dc-predictor-6b5b687b9f-dvfl2' metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:55:17Z" name: success-200-isvc-716dc-predictor-6b5b687b9f.18c78518677c7dd3 namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 3cabfcbb-e04e-46d2-a1c1-02b58f6bf77c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-716dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 102c7281-501c-4685-89b3-2dffa4c9d0df kind: Event lastTimestamp: "2026-07-31T23:55:17Z" message: Scaled up replica set success-200-isvc-716dc-predictor-6b5b687b9f from 0 to 1 metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:55:17Z" name: success-200-isvc-716dc-predictor.18c7851864fd0151 namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 796584a5-af53-4ce0-a5b8-0cb1c40e4617 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T23:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-716dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: 828c5caf-23ca-4783-afe4-3e985959bfbd kind: Event lastTimestamp: "2026-07-31T23:56:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:02Z" name: success-200-isvc-716dc-predictor.18c7851be377c7ed namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: 7d5437b6-d5cb-436f-8038-1a9815144d22 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T23:55:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-716dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: 828c5caf-23ca-4783-afe4-3e985959bfbd kind: Event lastTimestamp: "2026-07-31T23:56:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:02Z" name: success-200-isvc-716dc-predictor.18c7851be3d6e3b2 namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: 863f5978-48c4-42fe-91be-a5f33b8cb5fd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T23:56:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-716dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14655" uid: 828c5caf-23ca-4783-afe4-3e985959bfbd kind: Event lastTimestamp: "2026-07-31T23:56:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:32Z" name: success-200-isvc-716dc-predictor.18c7852660143631 namespace: kserve-ci-e2e-test resourceVersion: "15192" uid: b6c296f9-a3db-4063-8b7a-511d7e0c5fe1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T23:56:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-716dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "14655" uid: 828c5caf-23ca-4783-afe4-3e985959bfbd kind: Event lastTimestamp: "2026-07-31T23:56:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:32Z" name: success-200-isvc-716dc-predictor.18c785266089418a namespace: kserve-ci-e2e-test resourceVersion: "15193" uid: f2179443-b6c1-455c-827f-b5f5fa4dec04 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T23:55:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-716dc namespace: kserve-ci-e2e-test resourceVersion: "14368" uid: 7b9cb00a-14c4-4006-8ee4-33e8ebadb0d5 kind: Event lastTimestamp: "2026-07-31T23:56:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T23:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:56:19Z" name: success-200-isvc-716dc.18c7851865020501 namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: eae9b955-ad90-4790-95ac-13fa669aa620 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-716dc namespace: kserve-ci-e2e-test resourceVersion: "15089" uid: 7b9cb00a-14c4-4006-8ee4-33e8ebadb0d5 kind: Event lastTimestamp: "2026-07-31T23:56:19Z" message: InferenceService [success-200-isvc-716dc] is Ready metadata: creationTimestamp: "2026-07-31T23:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:56:19Z" name: success-200-isvc-716dc.18c78526d7389a3a namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: 9c76f0f0-6afd-42d1-be1a-deccf6de31c1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T23:57:37.227009Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 15f7d851-03e8-46bc-b59c-1b9ef87d6847 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T23:57:37Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4.18c7853905772a7c namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: 4882c0f4-3b3f-4770-a5a3-6fcf7a3cba9e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 15f7d851-03e8-46bc-b59c-1b9ef87d6847 kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a3ab7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:37Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4.18c7853914eba517 namespace: kserve-ci-e2e-test resourceVersion: "16122" uid: d07721e6-2a33-4c6d-bbcc-9603f4c3752c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: 15f7d851-03e8-46bc-b59c-1b9ef87d6847 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T23:57:38Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4.18c7853942f641a4 namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 959213a7-b95e-4ab8-abd7-901c04d136a7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 15f7d851-03e8-46bc-b59c-1b9ef87d6847 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:38Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4.18c7853944432075 namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: 7c4345d7-6966-404b-91be-37c72dc82cec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 15f7d851-03e8-46bc-b59c-1b9ef87d6847 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:38Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4.18c7853947a66fdc namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: e25ae822-8ce8-4829-8b6f-65bd5f5d46b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 15f7d851-03e8-46bc-b59c-1b9ef87d6847 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:38Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4.18c78539481cc01a namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: cee5c8c3-aab5-4d19-acbc-141b4d0d9556 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 15f7d851-03e8-46bc-b59c-1b9ef87d6847 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:38Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4.18c785394826f9ac namespace: kserve-ci-e2e-test resourceVersion: "16143" uid: 106fa0bb-0be6-437c-ae52-5b1e868aec18 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 15f7d851-03e8-46bc-b59c-1b9ef87d6847 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:38Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4.18c785394ad2520c namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: abb7c3cd-f7b9-42af-866d-6a22793e4f79 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 15f7d851-03e8-46bc-b59c-1b9ef87d6847 kind: Event lastTimestamp: "2026-07-31T23:57:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:38Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4.18c785394b47118b namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: b9f86874-314f-42fd-93cf-9bc00c197d06 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T23:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 15f7d851-03e8-46bc-b59c-1b9ef87d6847 kind: Event lastTimestamp: "2026-08-01T00:06:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T23:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:54Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4.18c785395a16bc90 namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: 3563b49b-bcfe-4ba8-b757-2766a0149d33 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 15f7d851-03e8-46bc-b59c-1b9ef87d6847 kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:52Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4.18c785ba37229749 namespace: kserve-ci-e2e-test resourceVersion: "20351" uid: f2e5c9d1-b4e5-4ad2-a647-58f2b6fc5a76 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 15f7d851-03e8-46bc-b59c-1b9ef87d6847 kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:52Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4.18c785ba37230f0e namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: c2649e35-0c85-4106-8582-a4ce798dffb6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 15f7d851-03e8-46bc-b59c-1b9ef87d6847 kind: Event lastTimestamp: "2026-08-01T00:06:54Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:54Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4.18c785bace7aa075 namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: e38a304f-e82d-4b6b-b005-f1639680a443 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a3ab7-predictor-7544d96d6f namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: c047b1cb-3a38-4fe7-b46d-b1e4e4ec2576 kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: 'Created pod: success-200-isvc-a3ab7-predictor-7544d96d6f-m4qb4' metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:37Z" name: success-200-isvc-a3ab7-predictor-7544d96d6f.18c7853904c5dc5d namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 2c090eed-cdd1-4df0-bce4-9852e2e515f9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a3ab7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: ebe0afc6-bd46-44bb-a7fc-2f85042eed8b kind: Event lastTimestamp: "2026-07-31T23:57:37Z" message: Scaled up replica set success-200-isvc-a3ab7-predictor-7544d96d6f from 0 to 1 metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:37Z" name: success-200-isvc-a3ab7-predictor.18c7853902ea46e4 namespace: kserve-ci-e2e-test resourceVersion: "16055" uid: cc3b93e5-437b-42f8-a85e-b0c80299b3b9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3ab7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: 096350e3-8a6e-4d9c-bec6-f7ed1c409b87 kind: Event lastTimestamp: "2026-07-31T23:57:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:52Z" name: success-200-isvc-a3ab7-predictor.18c7853c81505203 namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 0423f27d-c159-4b6d-bc99-d7100fe3f00e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3ab7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: 096350e3-8a6e-4d9c-bec6-f7ed1c409b87 kind: Event lastTimestamp: "2026-07-31T23:57:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:52Z" name: success-200-isvc-a3ab7-predictor.18c7853c81b03dc5 namespace: kserve-ci-e2e-test resourceVersion: "16324" uid: 4868fac6-0d0e-48fc-b162-86a3397090c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T23:58:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3ab7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16323" uid: 096350e3-8a6e-4d9c-bec6-f7ed1c409b87 kind: Event lastTimestamp: "2026-07-31T23:58:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:52Z" name: success-200-isvc-a3ab7-predictor.18c7854000514e00 namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: b18228ce-f4cf-4c44-87fb-ea7555d3b060 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T23:58:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3ab7-predictor namespace: kserve-ci-e2e-test resourceVersion: "16323" uid: 096350e3-8a6e-4d9c-bec6-f7ed1c409b87 kind: Event lastTimestamp: "2026-07-31T23:58:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T23:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:52Z" name: success-200-isvc-a3ab7-predictor.18c7854000bcdf54 namespace: kserve-ci-e2e-test resourceVersion: "16842" uid: c273cb28-1257-4b14-b6a0-36f58ae6c94f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T23:57:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3ab7 namespace: kserve-ci-e2e-test resourceVersion: "16027" uid: cc3dbb80-299f-4d41-b4a3-058a9557086f kind: Event lastTimestamp: "2026-07-31T23:58:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T23:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:58:24Z" name: success-200-isvc-a3ab7.18c78539030348b6 namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: 2012b850-5b5c-4f6e-8d99-4548914b74ac reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:58:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3ab7 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: cc3dbb80-299f-4d41-b4a3-058a9557086f kind: Event lastTimestamp: "2026-07-31T23:58:24Z" message: InferenceService [success-200-isvc-a3ab7] is Ready metadata: creationTimestamp: "2026-07-31T23:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:58:24Z" name: success-200-isvc-a3ab7.18c7854415244036 namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 6d6b3f32-7ed7-423d-9cdf-849a937a503d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:17:17.455079Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: c3f7ef2d-1ce3-4f69-9660-f559ac064ea7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bb639-predictor-7fccb6f469-zdmj7 to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:17:17Z" name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7.18c7864bd08a4b1f namespace: kserve-ci-e2e-test resourceVersion: "25917" uid: 17acf0c5-3317-4da8-863d-a96b2e9b1c27 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: c3f7ef2d-1ce3-4f69-9660-f559ac064ea7 kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-bb639-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:17Z" name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7.18c7864bdc35c346 namespace: kserve-ci-e2e-test resourceVersion: "25953" uid: 3ecd3c31-61a8-4d0f-a14a-2bb4e5b43cbf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7 namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: c3f7ef2d-1ce3-4f69-9660-f559ac064ea7 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: Add eth0 [10.133.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:17:18Z" name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7.18c7864c0e6836f3 namespace: kserve-ci-e2e-test resourceVersion: "25970" uid: fee1a3ac-97c2-4b4c-a825-8cdd50a0d7c5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: c3f7ef2d-1ce3-4f69-9660-f559ac064ea7 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:18Z" name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7.18c7864c0fa33606 namespace: kserve-ci-e2e-test resourceVersion: "25972" uid: 5f3220ba-e21b-4b26-a6e0-9bac7f2a2aeb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: c3f7ef2d-1ce3-4f69-9660-f559ac064ea7 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:18Z" name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7.18c7864c12fc0cd1 namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 33f0e7c9-d0b1-4e10-afdf-82a17b4b4619 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: c3f7ef2d-1ce3-4f69-9660-f559ac064ea7 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:18Z" name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7.18c7864c136fa918 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 52d67011-88ad-4e79-9512-ef2d60d39f66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: c3f7ef2d-1ce3-4f69-9660-f559ac064ea7 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:18Z" name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7.18c7864c137ad78c namespace: kserve-ci-e2e-test resourceVersion: "25976" uid: aa19c942-76ff-4159-bd3a-ffa76e5d8021 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: c3f7ef2d-1ce3-4f69-9660-f559ac064ea7 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:18Z" name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7.18c7864c16919e52 namespace: kserve-ci-e2e-test resourceVersion: "25979" uid: b5f83b98-6df2-4b84-8239-74b1b5216c53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: c3f7ef2d-1ce3-4f69-9660-f559ac064ea7 kind: Event lastTimestamp: "2026-08-01T00:17:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:18Z" name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7.18c7864c17087ba1 namespace: kserve-ci-e2e-test resourceVersion: "25981" uid: 7ff5b80e-5fbf-44e5-bc04-02c7f6f77d78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: c3f7ef2d-1ce3-4f69-9660-f559ac064ea7 kind: Event lastTimestamp: "2026-08-01T00:17:56Z" message: 'Readiness probe failed: dial tcp 10.133.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:56Z" name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7.18c7864c750b04e8 namespace: kserve-ci-e2e-test resourceVersion: "26509" uid: 370d6e8d-0387-4ef7-aa55-43aec8f3f59c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:34:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: c3f7ef2d-1ce3-4f69-9660-f559ac064ea7 kind: Event lastTimestamp: "2026-08-01T00:34:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:34:37Z" name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7.18c7873ddac8115b namespace: kserve-ci-e2e-test resourceVersion: "33600" uid: 98b23bc6-f720-4b65-aba0-f2aededafd66 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:34:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: c3f7ef2d-1ce3-4f69-9660-f559ac064ea7 kind: Event lastTimestamp: "2026-08-01T00:34:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:34:37Z" name: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7.18c7873ddac8b4aa namespace: kserve-ci-e2e-test resourceVersion: "33601" uid: b1c97e87-cc9d-4335-be56-7b1ee30754de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bb639-predictor-7fccb6f469 namespace: kserve-ci-e2e-test resourceVersion: "25897" uid: b1e9bff1-78ae-43bd-877f-10c0ec4ed1f5 kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: 'Created pod: success-200-isvc-bb639-predictor-7fccb6f469-zdmj7' metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:17Z" name: success-200-isvc-bb639-predictor-7fccb6f469.18c7864bcffdaed9 namespace: kserve-ci-e2e-test resourceVersion: "25913" uid: 97aab262-1736-465e-aae9-b05085b46906 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bb639-predictor namespace: kserve-ci-e2e-test resourceVersion: "25891" uid: 236e34b0-3107-4777-b401-200367704b94 kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: Scaled up replica set success-200-isvc-bb639-predictor-7fccb6f469 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:17Z" name: success-200-isvc-bb639-predictor.18c7864bce2584f5 namespace: kserve-ci-e2e-test resourceVersion: "25900" uid: 0ff8703e-7691-4c91-8f3f-c5815bc5af1e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bb639-predictor namespace: kserve-ci-e2e-test resourceVersion: "25893" uid: 5031e430-24f1-4fdc-8d30-52d71c72a4e5 kind: Event lastTimestamp: "2026-08-01T00:17:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:32Z" name: success-200-isvc-bb639-predictor.18c7864f4c8db1d9 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: 3d08fdbe-6f62-47bf-915f-1c8c514280bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bb639-predictor namespace: kserve-ci-e2e-test resourceVersion: "25893" uid: 5031e430-24f1-4fdc-8d30-52d71c72a4e5 kind: Event lastTimestamp: "2026-08-01T00:17:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:32Z" name: success-200-isvc-bb639-predictor.18c7864f4ceb4e94 namespace: kserve-ci-e2e-test resourceVersion: "26156" uid: c49bc087-2007-4e73-95cf-f2dc406739fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-01T00:17:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bb639-predictor namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: 5031e430-24f1-4fdc-8d30-52d71c72a4e5 kind: Event lastTimestamp: "2026-08-01T00:18:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:32Z" name: success-200-isvc-bb639-predictor.18c78652cbf8722a namespace: kserve-ci-e2e-test resourceVersion: "26866" uid: 49ab9378-e881-4813-bf87-ddf0bf092fc8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-01T00:17:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bb639-predictor namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: 5031e430-24f1-4fdc-8d30-52d71c72a4e5 kind: Event lastTimestamp: "2026-08-01T00:18:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:32Z" name: success-200-isvc-bb639-predictor.18c78652cc55c0dd namespace: kserve-ci-e2e-test resourceVersion: "26867" uid: 6e63fed7-7aaa-4fe3-9cf5-11712d3e932d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bb639 namespace: kserve-ci-e2e-test resourceVersion: "25871" uid: 8ebdadb7-f4d6-4d75-a97d-a027eddacf04 kind: Event lastTimestamp: "2026-08-01T00:18:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:18:06Z" name: success-200-isvc-bb639.18c7864bce710965 namespace: kserve-ci-e2e-test resourceVersion: "26598" uid: be9032a2-e26c-496d-b989-f75fb5268505 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bb639 namespace: kserve-ci-e2e-test resourceVersion: "25871" uid: 8ebdadb7-f4d6-4d75-a97d-a027eddacf04 kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: 'Failed to update status for InferenceService "success-200-isvc-bb639": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bb639": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:17:17Z" name: success-200-isvc-bb639.18c7864bced23151 namespace: kserve-ci-e2e-test resourceVersion: "25905" uid: 09086eae-3ee6-4da5-afcb-4003df2ca8cc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bb639 namespace: kserve-ci-e2e-test resourceVersion: "25871" uid: 8ebdadb7-f4d6-4d75-a97d-a027eddacf04 kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bb639": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:17:17Z" name: success-200-isvc-bb639.18c7864bced2855f namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: 9f79b83b-df94-4b56-bfc4-1b3e2c0ba78e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bb639 namespace: kserve-ci-e2e-test resourceVersion: "26596" uid: 8ebdadb7-f4d6-4d75-a97d-a027eddacf04 kind: Event lastTimestamp: "2026-08-01T00:18:06Z" message: InferenceService [success-200-isvc-bb639] is Ready metadata: creationTimestamp: "2026-08-01T00:18:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:18:06Z" name: success-200-isvc-bb639.18c78657301cc103 namespace: kserve-ci-e2e-test resourceVersion: "26597" uid: 8720af6a-6255-4da1-8f08-d94b6bd3559d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:16:27.699805Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g namespace: kserve-ci-e2e-test resourceVersion: "25082" uid: 9f543e2b-4d86-42f6-9859-dc6dab87f5b2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:16:27Z" name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g.18c786403ae5106e namespace: kserve-ci-e2e-test resourceVersion: "25088" uid: 76e70a77-392c-49c2-a32c-fcc1de3fa4bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g namespace: kserve-ci-e2e-test resourceVersion: "25103" uid: 9f543e2b-4d86-42f6-9859-dc6dab87f5b2 kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:16:28Z" name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g.18c7864054d6e9de namespace: kserve-ci-e2e-test resourceVersion: "25143" uid: cdc5b654-ce58-41ea-a5d3-bd47bb57ff4e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 9f543e2b-4d86-42f6-9859-dc6dab87f5b2 kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:28Z" name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g.18c7864056269a06 namespace: kserve-ci-e2e-test resourceVersion: "25145" uid: daf5df4d-dc2a-4745-9a74-8f1d9db8ad85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 9f543e2b-4d86-42f6-9859-dc6dab87f5b2 kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:28Z" name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g.18c78640591719b2 namespace: kserve-ci-e2e-test resourceVersion: "25147" uid: 07475dc6-adda-44d6-8bed-e892760caed0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 9f543e2b-4d86-42f6-9859-dc6dab87f5b2 kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:28Z" name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g.18c78640598c944f namespace: kserve-ci-e2e-test resourceVersion: "25149" uid: 76ef22c5-c958-442c-82ce-00f23092c12b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 9f543e2b-4d86-42f6-9859-dc6dab87f5b2 kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:28Z" name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g.18c786405997ccad namespace: kserve-ci-e2e-test resourceVersion: "25150" uid: d90284d1-fd34-43cb-b785-516741e78372 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 9f543e2b-4d86-42f6-9859-dc6dab87f5b2 kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:28Z" name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g.18c786405c9a0597 namespace: kserve-ci-e2e-test resourceVersion: "25153" uid: 8cffdb93-130a-499d-8e2f-0ca824c21290 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 9f543e2b-4d86-42f6-9859-dc6dab87f5b2 kind: Event lastTimestamp: "2026-08-01T00:16:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:16:28Z" name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g.18c786405d0add41 namespace: kserve-ci-e2e-test resourceVersion: "25154" uid: 7f00f480-37f4-4eae-9583-5ea23f441d0f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 9f543e2b-4d86-42f6-9859-dc6dab87f5b2 kind: Event lastTimestamp: "2026-08-01T00:17:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:04Z" name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g.18c7864085e16b63 namespace: kserve-ci-e2e-test resourceVersion: "25739" uid: 85fa2ec1-4712-4ca7-a997-7ccedb273786 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 9f543e2b-4d86-42f6-9859-dc6dab87f5b2 kind: Event lastTimestamp: "2026-08-01T00:17:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:37Z" name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g.18c7865096b30f92 namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: 3af8c736-90e7-4f05-a60b-23d5007c91cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 9f543e2b-4d86-42f6-9859-dc6dab87f5b2 kind: Event lastTimestamp: "2026-08-01T00:17:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:37Z" name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g.18c7865096b46597 namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 3f6262b9-c220-4571-9201-4c60ce2c786d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 9f543e2b-4d86-42f6-9859-dc6dab87f5b2 kind: Event lastTimestamp: "2026-08-01T00:17:39Z" message: 'Readiness probe failed: Get "https://10.133.0.45:8643/healthz": dial tcp 10.133.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:17:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:39Z" name: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g.18c786510e0bcffb namespace: kserve-ci-e2e-test resourceVersion: "26355" uid: 52f1b435-9af6-4707-9ce2-f9577823565d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d0074-predictor-5fc6bf94b7 namespace: kserve-ci-e2e-test resourceVersion: "25069" uid: aaedf06b-0c82-41d7-a5c6-4bc245e6ec26 kind: Event lastTimestamp: "2026-08-01T00:16:27Z" message: 'Created pod: success-200-isvc-d0074-predictor-5fc6bf94b7-fxc9g' metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:16:27Z" name: success-200-isvc-d0074-predictor-5fc6bf94b7.18c786403a590a5f namespace: kserve-ci-e2e-test resourceVersion: "25083" uid: f37afa45-7568-4b37-b39d-01af7c3ca343 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d0074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25063" uid: 52774092-5562-4b5e-b1c8-1aadc1cc3b97 kind: Event lastTimestamp: "2026-08-01T00:16:27Z" message: Scaled up replica set success-200-isvc-d0074-predictor-5fc6bf94b7 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:16:27Z" name: success-200-isvc-d0074-predictor.18c7864036fb6401 namespace: kserve-ci-e2e-test resourceVersion: "25072" uid: bdd734bd-c518-4372-ae24-97caadec778f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25068" uid: b4b98698-bc40-45a4-9abd-6fb3b2dafdbc kind: Event lastTimestamp: "2026-08-01T00:16:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:16:42Z" name: success-200-isvc-d0074-predictor.18c78643b607e27d namespace: kserve-ci-e2e-test resourceVersion: "25508" uid: d6d57c10-5056-4309-af3a-a31a0fe8874b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25068" uid: b4b98698-bc40-45a4-9abd-6fb3b2dafdbc kind: Event lastTimestamp: "2026-08-01T00:16:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:16:42Z" name: success-200-isvc-d0074-predictor.18c78643b66aa919 namespace: kserve-ci-e2e-test resourceVersion: "25510" uid: 7f786ec5-1e17-430a-8aba-749a142b8eec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:16:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25509" uid: b4b98698-bc40-45a4-9abd-6fb3b2dafdbc kind: Event lastTimestamp: "2026-08-01T00:17:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:27Z" name: success-200-isvc-d0074-predictor.18c7864735149d83 namespace: kserve-ci-e2e-test resourceVersion: "26083" uid: 327d4dd4-08db-4f89-bf04-8698077f661d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:16:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0074-predictor namespace: kserve-ci-e2e-test resourceVersion: "25509" uid: b4b98698-bc40-45a4-9abd-6fb3b2dafdbc kind: Event lastTimestamp: "2026-08-01T00:17:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:17:27Z" name: success-200-isvc-d0074-predictor.18c7864735586804 namespace: kserve-ci-e2e-test resourceVersion: "26085" uid: 37af280c-55c7-40e8-8f78-8e717d4e8e2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0074 namespace: kserve-ci-e2e-test resourceVersion: "25048" uid: 99cc8647-a97a-4a48-af23-b5a87fe90d61 kind: Event lastTimestamp: "2026-08-01T00:17:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:17:15Z" name: success-200-isvc-d0074.18c7864037360471 namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: 6b868b27-ba49-4fc4-9e09-3488a7873b8e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0074 namespace: kserve-ci-e2e-test resourceVersion: "25048" uid: 99cc8647-a97a-4a48-af23-b5a87fe90d61 kind: Event lastTimestamp: "2026-08-01T00:16:27Z" message: 'Failed to update status for InferenceService "success-200-isvc-d0074": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d0074": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:16:27Z" name: success-200-isvc-d0074.18c7864037b2ef40 namespace: kserve-ci-e2e-test resourceVersion: "25077" uid: 982631fc-b9de-4856-9473-dd015e316805 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:16:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0074 namespace: kserve-ci-e2e-test resourceVersion: "25048" uid: 99cc8647-a97a-4a48-af23-b5a87fe90d61 kind: Event lastTimestamp: "2026-08-01T00:16:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d0074": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:16:27Z" name: success-200-isvc-d0074.18c7864037b36817 namespace: kserve-ci-e2e-test resourceVersion: "25080" uid: 94f58ad4-c346-4813-9ddb-48438a8304ba reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0074 namespace: kserve-ci-e2e-test resourceVersion: "25821" uid: 99cc8647-a97a-4a48-af23-b5a87fe90d61 kind: Event lastTimestamp: "2026-08-01T00:17:15Z" message: InferenceService [success-200-isvc-d0074] is Ready metadata: creationTimestamp: "2026-08-01T00:17:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:17:15Z" name: success-200-isvc-d0074.18c7864b4102e042 namespace: kserve-ci-e2e-test resourceVersion: "25822" uid: 22351d9b-f11d-418c-b5aa-d164ae8a2e5f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:08:02.508411Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: 2da97b83-5b75-4fb7-912b-0e2dd8d1444d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:08:02Z" name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx.18c785ca9b2434e1 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 814b1c65-b77f-47fd-a5af-9936e3bfd2d6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 2da97b83-5b75-4fb7-912b-0e2dd8d1444d kind: Event lastTimestamp: "2026-08-01T00:08:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d99eb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:02Z" name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx.18c785caa7ded9cd namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: dd29f83b-2bd6-4025-b7fa-0f6742dcda38 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: 2da97b83-5b75-4fb7-912b-0e2dd8d1444d kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:08:03Z" name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx.18c785cad8a1f7fb namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 11243e8c-b24c-41d8-b268-1bf49ca6015e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 2da97b83-5b75-4fb7-912b-0e2dd8d1444d kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:03Z" name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx.18c785cada0a1bf4 namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 36478471-258d-4345-b767-87c09d01f7d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 2da97b83-5b75-4fb7-912b-0e2dd8d1444d kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:03Z" name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx.18c785cadd023c5f namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: d4de194c-e886-4187-af11-0576a0957550 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 2da97b83-5b75-4fb7-912b-0e2dd8d1444d kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:03Z" name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx.18c785cadd766207 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 6f39b47b-3de2-45ff-bd9a-8a486e065013 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 2da97b83-5b75-4fb7-912b-0e2dd8d1444d kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:03Z" name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx.18c785cadd80d5c8 namespace: kserve-ci-e2e-test resourceVersion: "21430" uid: 8a04cb6d-583f-4a7c-b8b7-7cf2354d3de8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 2da97b83-5b75-4fb7-912b-0e2dd8d1444d kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:03Z" name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx.18c785cae0c5eccb namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: f36f071d-3079-4194-aa6a-212596e92659 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 2da97b83-5b75-4fb7-912b-0e2dd8d1444d kind: Event lastTimestamp: "2026-08-01T00:08:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:03Z" name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx.18c785cae14a965d namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: 140cc229-1106-4d5e-b1c8-21d9f6ec1122 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:08:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 2da97b83-5b75-4fb7-912b-0e2dd8d1444d kind: Event lastTimestamp: "2026-08-01T00:08:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:41Z" name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx.18c785cb5446ad6d namespace: kserve-ci-e2e-test resourceVersion: "21787" uid: 49873a8a-0bee-4340-841c-f0004eca5a56 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 2da97b83-5b75-4fb7-912b-0e2dd8d1444d kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:17Z" name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx.18c7864bcb77ad02 namespace: kserve-ci-e2e-test resourceVersion: "25880" uid: f2a03973-51e0-400d-bfa2-1e6807b2323b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:17:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 2da97b83-5b75-4fb7-912b-0e2dd8d1444d kind: Event lastTimestamp: "2026-08-01T00:17:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:17:17Z" name: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx.18c7864bcb780f8a namespace: kserve-ci-e2e-test resourceVersion: "25877" uid: 144a5a5e-a583-49a1-ba3d-502f90bb022c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d99eb-predictor-5bf7f45f47 namespace: kserve-ci-e2e-test resourceVersion: "21343" uid: 8c5cb667-12d8-4f51-918e-2b8d89c6ecbe kind: Event lastTimestamp: "2026-08-01T00:08:02Z" message: 'Created pod: success-200-isvc-d99eb-predictor-5bf7f45f47-77mxx' metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:02Z" name: success-200-isvc-d99eb-predictor-5bf7f45f47.18c785ca9ab5f520 namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: f1c35ac1-e917-4352-858f-0e05073b58bc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d99eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: ae9e79e9-d7dd-4846-a031-a07eb3d565be kind: Event lastTimestamp: "2026-08-01T00:08:02Z" message: Scaled up replica set success-200-isvc-d99eb-predictor-5bf7f45f47 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:02Z" name: success-200-isvc-d99eb-predictor.18c785ca98ea6c81 namespace: kserve-ci-e2e-test resourceVersion: "21349" uid: dd603cce-dfc9-4903-970a-029f3fa8c357 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d99eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: dc833106-ad68-4b8f-b5ec-db469f846fa9 kind: Event lastTimestamp: "2026-08-01T00:08:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:17Z" name: success-200-isvc-d99eb-predictor.18c785ce175849e0 namespace: kserve-ci-e2e-test resourceVersion: "21588" uid: 554d2940-c179-4e8a-91e3-0570377027ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d99eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: dc833106-ad68-4b8f-b5ec-db469f846fa9 kind: Event lastTimestamp: "2026-08-01T00:08:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:08:17Z" name: success-200-isvc-d99eb-predictor.18c785ce17bc3084 namespace: kserve-ci-e2e-test resourceVersion: "21590" uid: d7821d04-f25f-4275-8c70-6461638e515f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-01T00:08:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d99eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21589" uid: dc833106-ad68-4b8f-b5ec-db469f846fa9 kind: Event lastTimestamp: "2026-08-01T00:09:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:09:17Z" name: success-200-isvc-d99eb-predictor.18c785d19670586b namespace: kserve-ci-e2e-test resourceVersion: "22101" uid: 7c6b634b-cf2a-47ba-ac59-89c34b341efa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-08-01T00:08:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d99eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21589" uid: dc833106-ad68-4b8f-b5ec-db469f846fa9 kind: Event lastTimestamp: "2026-08-01T00:09:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:09:17Z" name: success-200-isvc-d99eb-predictor.18c785d196d0d947 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: f74efd51-96c1-4bbc-8d11-12fd77c50290 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d99eb namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: f78ed3f6-c12a-49a4-8717-dc4c4db43bd2 kind: Event lastTimestamp: "2026-08-01T00:08:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:08:51Z" name: success-200-isvc-d99eb.18c785ca98fdde38 namespace: kserve-ci-e2e-test resourceVersion: "21881" uid: b733f177-b7a9-486b-b8b2-a438052bba9f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d99eb namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: f78ed3f6-c12a-49a4-8717-dc4c4db43bd2 kind: Event lastTimestamp: "2026-08-01T00:08:02Z" message: 'Failed to update status for InferenceService "success-200-isvc-d99eb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d99eb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:08:02Z" name: success-200-isvc-d99eb.18c785ca99c0b415 namespace: kserve-ci-e2e-test resourceVersion: "21355" uid: 60ac0346-57b1-4caa-aa68-f727fb7d1a92 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d99eb namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: f78ed3f6-c12a-49a4-8717-dc4c4db43bd2 kind: Event lastTimestamp: "2026-08-01T00:08:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d99eb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:08:02Z" name: success-200-isvc-d99eb.18c785ca99c1222e namespace: kserve-ci-e2e-test resourceVersion: "21358" uid: 109cd1f6-8211-4f85-a0d7-ea5e1863e842 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d99eb namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: f78ed3f6-c12a-49a4-8717-dc4c4db43bd2 kind: Event lastTimestamp: "2026-08-01T00:08:51Z" message: InferenceService [success-200-isvc-d99eb] is Ready metadata: creationTimestamp: "2026-08-01T00:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:08:51Z" name: success-200-isvc-d99eb.18c785d60f97fbed namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: ab30caf3-6b60-476c-b954-fada37464d2b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:06:52.200452Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: b5f4558a-d013-46cb-98b6-84f50810cdba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-df362-predictor-ddc8d9667-jd4hz to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:06:52Z" name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz.18c785ba3c75d196 namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: 36744cb6-a52c-4046-8f43-e64eef225112 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: b5f4558a-d013-46cb-98b6-84f50810cdba kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-df362-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:52Z" name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz.18c785ba49782e7f namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: 6710b4bc-5a44-4553-898b-00c4abcea0a9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz namespace: kserve-ci-e2e-test resourceVersion: "20394" uid: b5f4558a-d013-46cb-98b6-84f50810cdba kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:06:53Z" name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz.18c785ba7a1b1345 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 70363e90-ac01-406e-98fb-89c956553b3c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: b5f4558a-d013-46cb-98b6-84f50810cdba kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:53Z" name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz.18c785ba7b6c15b0 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: dc42514b-79e9-482f-b46b-b5f1321a8c8a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: b5f4558a-d013-46cb-98b6-84f50810cdba kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:53Z" name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz.18c785ba7f6007a9 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: 3c3be481-fdb6-4fc2-8e8c-0a89577385d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: b5f4558a-d013-46cb-98b6-84f50810cdba kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:53Z" name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz.18c785ba7fefe372 namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: ea08e202-44b9-4d4b-890d-2d554d6c2601 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: b5f4558a-d013-46cb-98b6-84f50810cdba kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:53Z" name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz.18c785ba7ffccc34 namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 9808a4e5-f6d6-4a69-a32f-0c5e04a401e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: b5f4558a-d013-46cb-98b6-84f50810cdba kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:53Z" name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz.18c785ba82f8a3ed namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 561fabcc-95b5-433a-b085-93a6c469e1eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: b5f4558a-d013-46cb-98b6-84f50810cdba kind: Event lastTimestamp: "2026-08-01T00:06:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:53Z" name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz.18c785ba836b134c namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: f5ee6fce-770d-4923-bd33-14e26fa9ae9a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:06:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: b5f4558a-d013-46cb-98b6-84f50810cdba kind: Event lastTimestamp: "2026-08-01T00:07:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:31Z" name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz.18c785baf26650c2 namespace: kserve-ci-e2e-test resourceVersion: "20973" uid: 1a64c47f-2d44-4ee1-a998-431797c404c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: b5f4558a-d013-46cb-98b6-84f50810cdba kind: Event lastTimestamp: "2026-08-01T00:08:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:02Z" name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz.18c785ca96e7d4ea namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: e57202f2-76cf-499a-b783-b73a76584cc8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: b5f4558a-d013-46cb-98b6-84f50810cdba kind: Event lastTimestamp: "2026-08-01T00:08:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:08:02Z" name: success-200-isvc-df362-predictor-ddc8d9667-jd4hz.18c785ca96e8502f namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: 12060db4-ea38-4407-9e9c-5711e1a9174d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-df362-predictor-ddc8d9667 namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: 6b514b0d-6185-475f-9499-34de3b6b483c kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: 'Created pod: success-200-isvc-df362-predictor-ddc8d9667-jd4hz' metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:06:52Z" name: success-200-isvc-df362-predictor-ddc8d9667.18c785ba3c08a6ab namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: 6241424d-caae-4407-86a0-8238481bd509 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-df362-predictor namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 539da796-8cab-418f-af35-7c63f69d4208 kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: Scaled up replica set success-200-isvc-df362-predictor-ddc8d9667 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:06:52Z" name: success-200-isvc-df362-predictor.18c785ba3a32d76c namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: f69ef631-638a-428f-b8fc-9f1581eecf23 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:07:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df362-predictor namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 6c033677-b232-47c5-b12f-0de9d06bc17d kind: Event lastTimestamp: "2026-08-01T00:07:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:22Z" name: success-200-isvc-df362-predictor.18c785bdb8fa21c0 namespace: kserve-ci-e2e-test resourceVersion: "20898" uid: adaa64ea-96ad-402e-beb4-c7b6ba3b2dda reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:07:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df362-predictor namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 6c033677-b232-47c5-b12f-0de9d06bc17d kind: Event lastTimestamp: "2026-08-01T00:07:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:22Z" name: success-200-isvc-df362-predictor.18c785bdb95d7cbb namespace: kserve-ci-e2e-test resourceVersion: "20899" uid: 1a00d19b-0e01-4f1e-beac-948103863393 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:07:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df362-predictor namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 6c033677-b232-47c5-b12f-0de9d06bc17d kind: Event lastTimestamp: "2026-08-01T00:07:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:52Z" name: success-200-isvc-df362-predictor.18c785c4b6e863da namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: 6ebd07b0-f87a-48ff-a847-8a37eb422eaf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:07:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df362-predictor namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 6c033677-b232-47c5-b12f-0de9d06bc17d kind: Event lastTimestamp: "2026-08-01T00:07:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:52Z" name: success-200-isvc-df362-predictor.18c785c4b751156f namespace: kserve-ci-e2e-test resourceVersion: "21177" uid: 7197c6c1-4b56-48a4-9046-55eb69c38e0f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df362 namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: 61697cd6-f64b-4808-9bc0-79b95ac9e4e9 kind: Event lastTimestamp: "2026-08-01T00:07:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:07:41Z" name: success-200-isvc-df362.18c785ba3ab171b5 namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: 8bd0294f-ce07-4c98-a994-41ebcf32d58b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df362 namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: 61697cd6-f64b-4808-9bc0-79b95ac9e4e9 kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: 'Failed to update status for InferenceService "success-200-isvc-df362": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-df362": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:06:52Z" name: success-200-isvc-df362.18c785ba3b2322ec namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: c85a5bdc-4ecd-4a98-84ba-8082d731bc3b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df362 namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: 61697cd6-f64b-4808-9bc0-79b95ac9e4e9 kind: Event lastTimestamp: "2026-08-01T00:06:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-df362": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-08-01T00:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:06:52Z" name: success-200-isvc-df362.18c785ba3b238948 namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 27bba731-ac3e-4c8f-aa3e-baf979a1d89a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df362 namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: 61697cd6-f64b-4808-9bc0-79b95ac9e4e9 kind: Event lastTimestamp: "2026-08-01T00:07:41Z" message: InferenceService [success-200-isvc-df362] is Ready metadata: creationTimestamp: "2026-08-01T00:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:07:41Z" name: success-200-isvc-df362.18c785c5ada4b14e namespace: kserve-ci-e2e-test resourceVersion: "21062" uid: 229db7a1-2098-4337-b3ed-58795d95bea4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:06:02.357451Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276 namespace: kserve-ci-e2e-test resourceVersion: "19850" uid: cde6e7e4-4775-491c-8965-25508d23d619 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fe2ca-predictor-698655d7c5-6h276 to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:06:02Z" name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276.18c785aea1960109 namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: aca9bae5-0637-437f-b513-a0fa8a02f449 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276 namespace: kserve-ci-e2e-test resourceVersion: "19866" uid: cde6e7e4-4775-491c-8965-25508d23d619 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:06:02Z" name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276.18c785aebc3142b4 namespace: kserve-ci-e2e-test resourceVersion: "19902" uid: 836fc1cc-1d77-4bb7-88e0-e7b331b93deb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: cde6e7e4-4775-491c-8965-25508d23d619 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1841" already present on machine metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276.18c785aebd8e324a namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 02b12212-ff1f-47cc-bb4f-c289b066e384 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: cde6e7e4-4775-491c-8965-25508d23d619 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276.18c785aec0abc043 namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: e435d499-77e0-4b18-bb21-457b934f2a13 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: cde6e7e4-4775-491c-8965-25508d23d619 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276.18c785aec1329591 namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: 5e65bc82-3ba5-4b1b-be37-11a31627cba4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: cde6e7e4-4775-491c-8965-25508d23d619 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276.18c785aec13d8cff namespace: kserve-ci-e2e-test resourceVersion: "19907" uid: c429316f-2fca-4f56-8f11-9f964472b993 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: cde6e7e4-4775-491c-8965-25508d23d619 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276.18c785aec48a81ce namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 73796f74-fed0-4e5d-bcea-351bbe891505 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: cde6e7e4-4775-491c-8965-25508d23d619 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276.18c785aec50438f7 namespace: kserve-ci-e2e-test resourceVersion: "19912" uid: 0a1b09ce-f9be-4904-8bc7-1d290e0cca26 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: cde6e7e4-4775-491c-8965-25508d23d619 kind: Event lastTimestamp: "2026-08-01T00:06:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:40Z" name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276.18c785af03a5f08c namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: ec7f2023-e9a1-4ff1-8c2c-d0a5cbae0ddf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: cde6e7e4-4775-491c-8965-25508d23d619 kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:12Z" name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276.18c785befa88cd3d namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: 2afdfbbb-6f80-4aa8-a017-b9154bf5fac4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: cde6e7e4-4775-491c-8965-25508d23d619 kind: Event lastTimestamp: "2026-08-01T00:07:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-08-01T00:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:12Z" name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276.18c785befa896f91 namespace: kserve-ci-e2e-test resourceVersion: "20701" uid: 057db321-14e6-4d57-9c38-0d6df8f4aece reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:07:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: cde6e7e4-4775-491c-8965-25508d23d619 kind: Event lastTimestamp: "2026-08-01T00:07:15Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-08-01T00:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:07:15Z" name: success-200-isvc-fe2ca-predictor-698655d7c5-6h276.18c785bf8bc8dba0 namespace: kserve-ci-e2e-test resourceVersion: "20823" uid: 1c0493c9-e77f-444b-ae68-7fcb22a26023 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fe2ca-predictor-698655d7c5 namespace: kserve-ci-e2e-test resourceVersion: "19836" uid: 33cdfec1-c800-4b9c-bb19-25450b07c310 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: 'Created pod: success-200-isvc-fe2ca-predictor-698655d7c5-6h276' metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:06:02Z" name: success-200-isvc-fe2ca-predictor-698655d7c5.18c785aea11d2fb2 namespace: kserve-ci-e2e-test resourceVersion: "19855" uid: 0e4de208-bdd3-4597-b3df-0fbec8ccef34 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fe2ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19834" uid: 962e92b0-ae02-4792-855d-f0ea92b0f3e7 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Scaled up replica set success-200-isvc-fe2ca-predictor-698655d7c5 from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:06:02Z" name: success-200-isvc-fe2ca-predictor.18c785ae9de2d749 namespace: kserve-ci-e2e-test resourceVersion: "19837" uid: 88f200e1-eac5-4eac-9f11-7615c2d33cb0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fe2ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19835" uid: d28c1025-0e39-4786-a1e2-adbccf256be3 kind: Event lastTimestamp: "2026-08-01T00:06:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:06:47Z" name: success-200-isvc-fe2ca-predictor.18c785b21cdd5c5c namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 1d38131d-e2e5-4041-b277-2632cb067688 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fe2ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19835" uid: d28c1025-0e39-4786-a1e2-adbccf256be3 kind: Event lastTimestamp: "2026-08-01T00:06:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:06:47Z" name: success-200-isvc-fe2ca-predictor.18c785b21d4696e3 namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: 657453b1-584a-41f4-a78a-fb82d96ebeb0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:06:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fe2ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "20038" uid: d28c1025-0e39-4786-a1e2-adbccf256be3 kind: Event lastTimestamp: "2026-08-01T00:07:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:02Z" name: success-200-isvc-fe2ca-predictor.18c785b59bfb4f54 namespace: kserve-ci-e2e-test resourceVersion: "20563" uid: f2cb79fc-3edc-4f11-999d-0df267cd0b7b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-08-01T00:06:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fe2ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "20038" uid: d28c1025-0e39-4786-a1e2-adbccf256be3 kind: Event lastTimestamp: "2026-08-01T00:07:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-08-01T00:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:07:02Z" name: success-200-isvc-fe2ca-predictor.18c785b59c64fb6a namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: 9b5648d7-2b6d-40b3-825a-c002564709ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fe2ca namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: e24fa4e1-9528-4083-a1f6-4762d0cc701c kind: Event lastTimestamp: "2026-08-01T00:06:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:06:50Z" name: success-200-isvc-fe2ca.18c785ae9de91483 namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: c8083df9-e373-4d44-88c1-fb0fe771507e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fe2ca namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: e24fa4e1-9528-4083-a1f6-4762d0cc701c kind: Event lastTimestamp: "2026-08-01T00:06:50Z" message: InferenceService [success-200-isvc-fe2ca] is Ready metadata: creationTimestamp: "2026-08-01T00:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:06:50Z" name: success-200-isvc-fe2ca.18c785b9beccd6df namespace: kserve-ci-e2e-test resourceVersion: "20299" uid: 53e7f78f-bf91-47db-b556-1e0e1b457f21 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T23:57:47.273200Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-32541-775bd66797-x4tlp namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 28872143-734e-478c-9000-be449b6d1734 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-32541-775bd66797-x4tlp to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-07-31T23:57:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T23:57:47Z" name: switch-graph-32541-775bd66797-x4tlp.18c7853b5c43dd0a namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 4d33049c-38d2-4acc-a526-5154d16d0c42 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:47Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-32541-775bd66797-x4tlp namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: 28872143-734e-478c-9000-be449b6d1734 kind: Event lastTimestamp: "2026-07-31T23:57:47Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T23:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T23:57:47Z" name: switch-graph-32541-775bd66797-x4tlp.18c7853b76770919 namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: 38ce60f6-950e-4d3d-b8bb-cbaf97550d91 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32541} kind: Pod name: switch-graph-32541-775bd66797-x4tlp namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 28872143-734e-478c-9000-be449b6d1734 kind: Event lastTimestamp: "2026-07-31T23:57:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:56dbaa30789264fbe80634bdc08e7f43bcfc19e5006180ec98cfd4cb6b8d99d6" already present on machine metadata: creationTimestamp: "2026-07-31T23:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:47Z" name: switch-graph-32541-775bd66797-x4tlp.18c7853b77b339d7 namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: da85cee6-4e15-4f71-a860-ff1e7fe6457d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32541} kind: Pod name: switch-graph-32541-775bd66797-x4tlp namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 28872143-734e-478c-9000-be449b6d1734 kind: Event lastTimestamp: "2026-07-31T23:57:47Z" message: 'Created container: switch-graph-32541' metadata: creationTimestamp: "2026-07-31T23:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:47Z" name: switch-graph-32541-775bd66797-x4tlp.18c7853b7aedc7d0 namespace: kserve-ci-e2e-test resourceVersion: "16285" uid: b6c3c136-1617-4700-887f-7c3d957a022c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32541} kind: Pod name: switch-graph-32541-775bd66797-x4tlp namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 28872143-734e-478c-9000-be449b6d1734 kind: Event lastTimestamp: "2026-07-31T23:57:47Z" message: Started container switch-graph-32541 metadata: creationTimestamp: "2026-07-31T23:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:47Z" name: switch-graph-32541-775bd66797-x4tlp.18c7853b7b698b41 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 1779cac0-1f34-4e9b-a94e-b4d4cff79319 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32541} kind: Pod name: switch-graph-32541-775bd66797-x4tlp namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 28872143-734e-478c-9000-be449b6d1734 kind: Event lastTimestamp: "2026-08-01T00:06:02Z" message: Stopping container switch-graph-32541 metadata: creationTimestamp: "2026-08-01T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:02Z" name: switch-graph-32541-775bd66797-x4tlp.18c785ae927fdeae namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 0e041467-776f-402b-94d7-b854cc896175 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-08-01T00:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-32541} kind: Pod name: switch-graph-32541-775bd66797-x4tlp namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 28872143-734e-478c-9000-be449b6d1734 kind: Event lastTimestamp: "2026-08-01T00:06:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-01T00:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:06:29Z" name: switch-graph-32541-775bd66797-x4tlp.18c785af2d61bf3d namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: b712ed4d-7e5d-4998-afa0-9ef3f4a16a81 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-32541-775bd66797 namespace: kserve-ci-e2e-test resourceVersion: "16251" uid: 0d6960ba-8641-4d3d-b156-54fbac457517 kind: Event lastTimestamp: "2026-07-31T23:57:47Z" message: 'Created pod: switch-graph-32541-775bd66797-x4tlp' metadata: creationTimestamp: "2026-07-31T23:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:47Z" name: switch-graph-32541-775bd66797.18c7853b5bdf0146 namespace: kserve-ci-e2e-test resourceVersion: "16258" uid: be3d10e1-af6a-40fb-8e57-87ba3ac06563 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-32541 namespace: kserve-ci-e2e-test resourceVersion: "16249" uid: 10dba798-7830-48e5-95ad-004099d39376 kind: Event lastTimestamp: "2026-07-31T23:57:47Z" message: Scaled up replica set switch-graph-32541-775bd66797 from 0 to 1 metadata: creationTimestamp: "2026-07-31T23:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:57:47Z" name: switch-graph-32541.18c7853b5b33dfe3 namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: de27af60-16d9-48b4-944d-0a86881f271e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-32541 namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 0886b324-16ac-4a6c-b256-7e8b09008d3a kind: Event lastTimestamp: "2026-07-31T23:57:47Z" message: InferenceGraph [switch-graph-32541] is Ready metadata: creationTimestamp: "2026-07-31T23:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:47Z" name: switch-graph-32541.18c7853b5f1516d0 namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: ed91982c-9ed2-4e10-9d9c-19ae16f2b646 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-32541 namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: 0886b324-16ac-4a6c-b256-7e8b09008d3a kind: Event lastTimestamp: "2026-07-31T23:57:47Z" message: 'Failed to update status for InferenceGraph "switch-graph-32541": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-32541": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T23:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:47Z" name: switch-graph-32541.18c7853b6150d0c1 namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: 2d30160f-be39-414c-8bb7-2e491fd4cdc7 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:57:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-32541 namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: 0886b324-16ac-4a6c-b256-7e8b09008d3a kind: Event lastTimestamp: "2026-07-31T23:57:47Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-32541": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T23:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:57:47Z" name: switch-graph-32541.18c7853b61511f71 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: bec1a413-20a2-4fb6-8997-c9ce1610a789 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T23:58:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-32541 namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: 9d34a1dd-bdb9-427c-be72-6d5c286901c1 kind: Event lastTimestamp: "2026-07-31T23:58:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:32Z" name: switch-graph-32541.18c7853ed9ab5666 namespace: kserve-ci-e2e-test resourceVersion: "16653" uid: 36eb11f6-2777-4655-85de-e8692e0ac76c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T23:58:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-32541 namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: 9d34a1dd-bdb9-427c-be72-6d5c286901c1 kind: Event lastTimestamp: "2026-07-31T23:58:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T23:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:58:32Z" name: switch-graph-32541.18c7853eda0b3b3e namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: 32ffde3d-8541-4c8e-9442-56726231e9b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-31T23:56:36.885347Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-716dc-7d7ffc79d4-2bdn6 namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: 161566e9-254c-4c2d-aa6c-351f0d666a07 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-716dc-7d7ffc79d4-2bdn6 to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-07-31T23:56:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T23:56:36Z" name: switch-graph-716dc-7d7ffc79d4-2bdn6.18c7852af8d279fc namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 096d0c5f-6c27-46b8-b41d-23209e7d7e91 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:37Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-716dc-7d7ffc79d4-2bdn6 namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: 161566e9-254c-4c2d-aa6c-351f0d666a07 kind: Event lastTimestamp: "2026-07-31T23:56:37Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T23:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T23:56:37Z" name: switch-graph-716dc-7d7ffc79d4-2bdn6.18c7852b128392d4 namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 5ebef207-6ef9-4eaa-b9fb-42f4c7c4a2e3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-716dc} kind: Pod name: switch-graph-716dc-7d7ffc79d4-2bdn6 namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 161566e9-254c-4c2d-aa6c-351f0d666a07 kind: Event lastTimestamp: "2026-07-31T23:56:37Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:56dbaa30789264fbe80634bdc08e7f43bcfc19e5006180ec98cfd4cb6b8d99d6" metadata: creationTimestamp: "2026-07-31T23:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:37Z" name: switch-graph-716dc-7d7ffc79d4-2bdn6.18c7852b13b544fa namespace: kserve-ci-e2e-test resourceVersion: "15274" uid: f95f56cf-7c97-4d3b-b4f9-1c5ac56eb779 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-716dc} kind: Pod name: switch-graph-716dc-7d7ffc79d4-2bdn6 namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 161566e9-254c-4c2d-aa6c-351f0d666a07 kind: Event lastTimestamp: "2026-07-31T23:56:39Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:56dbaa30789264fbe80634bdc08e7f43bcfc19e5006180ec98cfd4cb6b8d99d6" in 1.91s (1.91s including waiting). Image size: 218232958 bytes.' metadata: creationTimestamp: "2026-07-31T23:56:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:39Z" name: switch-graph-716dc-7d7ffc79d4-2bdn6.18c7852b85981629 namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 3648381a-017f-4fae-aa1e-b811af4f9704 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-716dc} kind: Pod name: switch-graph-716dc-7d7ffc79d4-2bdn6 namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 161566e9-254c-4c2d-aa6c-351f0d666a07 kind: Event lastTimestamp: "2026-07-31T23:56:39Z" message: 'Created container: switch-graph-716dc' metadata: creationTimestamp: "2026-07-31T23:56:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:39Z" name: switch-graph-716dc-7d7ffc79d4-2bdn6.18c7852b88a7bbe0 namespace: kserve-ci-e2e-test resourceVersion: "15289" uid: 31158dc3-419d-4cf9-be11-d3cabe40db03 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-716dc} kind: Pod name: switch-graph-716dc-7d7ffc79d4-2bdn6 namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 161566e9-254c-4c2d-aa6c-351f0d666a07 kind: Event lastTimestamp: "2026-07-31T23:56:39Z" message: Started container switch-graph-716dc metadata: creationTimestamp: "2026-07-31T23:56:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:39Z" name: switch-graph-716dc-7d7ffc79d4-2bdn6.18c7852b891b3942 namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: 87d29bed-b369-4095-95d3-649ee70b06fd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-716dc} kind: Pod name: switch-graph-716dc-7d7ffc79d4-2bdn6 namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 161566e9-254c-4c2d-aa6c-351f0d666a07 kind: Event lastTimestamp: "2026-07-31T23:56:47Z" message: Stopping container switch-graph-716dc metadata: creationTimestamp: "2026-07-31T23:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:56:47Z" name: switch-graph-716dc-7d7ffc79d4-2bdn6.18c7852d55ce9d15 namespace: kserve-ci-e2e-test resourceVersion: "15368" uid: 2d1f44b8-66eb-4689-a2c4-890fd0465277 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T23:56:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-716dc} kind: Pod name: switch-graph-716dc-7d7ffc79d4-2bdn6 namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 161566e9-254c-4c2d-aa6c-351f0d666a07 kind: Event lastTimestamp: "2026-07-31T23:57:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-31T23:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T23:57:15Z" name: switch-graph-716dc-7d7ffc79d4-2bdn6.18c7852e1c5a5178 namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: 6943a405-02f2-463c-9170-7ad8a88a9c41 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-716dc-7d7ffc79d4 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: 5716c771-55a9-4c67-bce0-78e5ffbcdc11 kind: Event lastTimestamp: "2026-07-31T23:56:36Z" message: 'Created pod: switch-graph-716dc-7d7ffc79d4-2bdn6' metadata: creationTimestamp: "2026-07-31T23:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:36Z" name: switch-graph-716dc-7d7ffc79d4.18c7852af86f3134 namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: 34aa9ee1-42c1-4b9d-8f0b-d04a3785be4b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-716dc namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: e8ef1cc7-a773-4c12-a84c-93b03158250c kind: Event lastTimestamp: "2026-07-31T23:56:36Z" message: Scaled up replica set switch-graph-716dc-7d7ffc79d4 from 0 to 1 metadata: creationTimestamp: "2026-07-31T23:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T23:56:36Z" name: switch-graph-716dc.18c7852af7ac14d0 namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: 477a38e7-c501-49b5-809b-7c644611135c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T23:56:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-716dc namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: 88663e5f-71f4-4b2a-992d-f2b2e7ff8c44 kind: Event lastTimestamp: "2026-07-31T23:56:36Z" message: InferenceGraph [switch-graph-716dc] is Ready metadata: creationTimestamp: "2026-07-31T23:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T23:56:36Z" name: switch-graph-716dc.18c7852afd307845 namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 8fc016be-3dbc-45e6-b59e-ec6f873858b5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-08-01T00:18:17.542510Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-bb639-658b9f69db-rwxvz namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 141f7de1-1db2-49fa-89d9-5cde41749e2b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-bb639-658b9f69db-rwxvz to ip-10-0-136-108.ec2.internal metadata: creationTimestamp: "2026-08-01T00:18:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-08-01T00:18:17Z" name: switch-graph-bb639-658b9f69db-rwxvz.18c78659ce07c15c namespace: kserve-ci-e2e-test resourceVersion: "26712" uid: 97ed1410-cbe6-45ab-8dec-aa9a2fbb297c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bb6bc6b65-jgn7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:17Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-bb639-658b9f69db-rwxvz namespace: kserve-ci-e2e-test resourceVersion: "26709" uid: 141f7de1-1db2-49fa-89d9-5cde41749e2b kind: Event lastTimestamp: "2026-08-01T00:18:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-bb639-serving-cert" not found' metadata: creationTimestamp: "2026-08-01T00:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:18:17Z" name: switch-graph-bb639-658b9f69db-rwxvz.18c78659dcb9c586 namespace: kserve-ci-e2e-test resourceVersion: "26725" uid: 2b568db4-3e63-4e99-b29b-72a3e2ba922b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:18Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-bb639-658b9f69db-rwxvz namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 141f7de1-1db2-49fa-89d9-5cde41749e2b kind: Event lastTimestamp: "2026-08-01T00:18:18Z" message: Add eth0 [10.133.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-08-01T00:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-08-01T00:18:18Z" name: switch-graph-bb639-658b9f69db-rwxvz.18c7865a0b8eb375 namespace: kserve-ci-e2e-test resourceVersion: "26737" uid: 86944a6d-a6f7-4fe1-8efe-53edb561a4f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bb639} kind: Pod name: switch-graph-bb639-658b9f69db-rwxvz namespace: kserve-ci-e2e-test resourceVersion: "26709" uid: 141f7de1-1db2-49fa-89d9-5cde41749e2b kind: Event lastTimestamp: "2026-08-01T00:18:18Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:56dbaa30789264fbe80634bdc08e7f43bcfc19e5006180ec98cfd4cb6b8d99d6" already present on machine metadata: creationTimestamp: "2026-08-01T00:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:18:18Z" name: switch-graph-bb639-658b9f69db-rwxvz.18c7865a0cc49c48 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: 13872c26-6d52-4d95-972c-8747b70caa43 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bb639} kind: Pod name: switch-graph-bb639-658b9f69db-rwxvz namespace: kserve-ci-e2e-test resourceVersion: "26709" uid: 141f7de1-1db2-49fa-89d9-5cde41749e2b kind: Event lastTimestamp: "2026-08-01T00:18:18Z" message: 'Created container: switch-graph-bb639' metadata: creationTimestamp: "2026-08-01T00:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:18:18Z" name: switch-graph-bb639-658b9f69db-rwxvz.18c7865a10a10a78 namespace: kserve-ci-e2e-test resourceVersion: "26741" uid: 14629330-8f18-461e-9d72-5cbcda11bc83 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bb639} kind: Pod name: switch-graph-bb639-658b9f69db-rwxvz namespace: kserve-ci-e2e-test resourceVersion: "26709" uid: 141f7de1-1db2-49fa-89d9-5cde41749e2b kind: Event lastTimestamp: "2026-08-01T00:18:18Z" message: Started container switch-graph-bb639 metadata: creationTimestamp: "2026-08-01T00:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:18:18Z" name: switch-graph-bb639-658b9f69db-rwxvz.18c7865a111423b1 namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: d3b98907-f294-45fc-9a93-eb57d075080c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:34:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bb639} kind: Pod name: switch-graph-bb639-658b9f69db-rwxvz namespace: kserve-ci-e2e-test resourceVersion: "26709" uid: 141f7de1-1db2-49fa-89d9-5cde41749e2b kind: Event lastTimestamp: "2026-08-01T00:34:36Z" message: Stopping container switch-graph-bb639 metadata: creationTimestamp: "2026-08-01T00:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:34:36Z" name: switch-graph-bb639-658b9f69db-rwxvz.18c7873dd1ef217e namespace: kserve-ci-e2e-test resourceVersion: "33584" uid: 1345f2d4-699d-4d1e-9002-5fb96721796c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-08-01T00:34:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bb639} kind: Pod name: switch-graph-bb639-658b9f69db-rwxvz namespace: kserve-ci-e2e-test resourceVersion: "26709" uid: 141f7de1-1db2-49fa-89d9-5cde41749e2b kind: Event lastTimestamp: "2026-08-01T00:35:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-08-01T00:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-08-01T00:35:00Z" name: switch-graph-bb639-658b9f69db-rwxvz.18c7873ead23d85c namespace: kserve-ci-e2e-test resourceVersion: "33828" uid: 5151e28d-d3e9-4a20-a95e-61f6962773bb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-108.ec2.internal source: component: kubelet host: ip-10-0-136-108.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-bb639-658b9f69db namespace: kserve-ci-e2e-test resourceVersion: "26704" uid: d5e5c2ba-db17-4b5a-a78a-2e45c5d11f81 kind: Event lastTimestamp: "2026-08-01T00:18:17Z" message: 'Created pod: switch-graph-bb639-658b9f69db-rwxvz' metadata: creationTimestamp: "2026-08-01T00:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:17Z" name: switch-graph-bb639-658b9f69db.18c78659cd8e470e namespace: kserve-ci-e2e-test resourceVersion: "26710" uid: 38c3c11e-9c65-411f-b9cb-5434e49cb025 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-bb639 namespace: kserve-ci-e2e-test resourceVersion: "26702" uid: 6e17b596-64db-4b94-bdf9-912523f75a61 kind: Event lastTimestamp: "2026-08-01T00:18:17Z" message: Scaled up replica set switch-graph-bb639-658b9f69db from 0 to 1 metadata: creationTimestamp: "2026-08-01T00:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:18:17Z" name: switch-graph-bb639.18c78659ccbd78ea namespace: kserve-ci-e2e-test resourceVersion: "26705" uid: 5774586e-4e29-4164-adef-65fbba8f75b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-08-01T00:18:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-bb639 namespace: kserve-ci-e2e-test resourceVersion: "26721" uid: 68f75dae-da63-43d7-8dd5-4a008706c99f kind: Event lastTimestamp: "2026-08-01T00:18:17Z" message: InferenceGraph [switch-graph-bb639] is Ready metadata: creationTimestamp: "2026-08-01T00:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-08-01T00:18:17Z" name: switch-graph-bb639.18c78659d1fad9f4 namespace: kserve-ci-e2e-test resourceVersion: "26723" uid: bc20d9ea-887e-47d9-8fbb-9792c219744b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:18:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-bb639 namespace: kserve-ci-e2e-test resourceVersion: "26703" uid: 5262de21-74fd-4c49-b369-3004c6959fd6 kind: Event lastTimestamp: "2026-08-01T00:19:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:19:02Z" name: switch-graph-bb639.18c7865d4b2718b6 namespace: kserve-ci-e2e-test resourceVersion: "27132" uid: b24f5a2f-3107-495e-946d-5645f10b4d33 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-08-01T00:18:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-bb639 namespace: kserve-ci-e2e-test resourceVersion: "26703" uid: 5262de21-74fd-4c49-b369-3004c6959fd6 kind: Event lastTimestamp: "2026-08-01T00:19:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-08-01T00:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-08-01T00:19:02Z" name: switch-graph-bb639.18c7865d4b8784d6 namespace: kserve-ci-e2e-test resourceVersion: "27133" uid: 7ed81826-1381-4269-b3bd-51573e37627d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33862"