--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:06:58.035422Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-cf436-7d4dbbfd97-vqcsn namespace: kserve-ci-e2e-test resourceVersion: "20379" uid: bf4f73ea-b479-4bfa-a4b3-5d4dc3a89ad3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-cf436-7d4dbbfd97-vqcsn to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-07-15T11:06:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:06:58Z" name: ensemble-graph-cf436-7d4dbbfd97-vqcsn.18c271e209f04c7f namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: 28f201ec-a702-43b4-9297-f32f83344883 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:58Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-cf436-7d4dbbfd97-vqcsn namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: bf4f73ea-b479-4bfa-a4b3-5d4dc3a89ad3 kind: Event lastTimestamp: "2026-07-15T11:06:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-cf436-serving-cert" not found' metadata: creationTimestamp: "2026-07-15T11:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:58Z" name: ensemble-graph-cf436-7d4dbbfd97-vqcsn.18c271e2175a53a6 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: e8b3294c-b51f-4d79-99b4-ad598fa82572 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:59Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-cf436-7d4dbbfd97-vqcsn namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: bf4f73ea-b479-4bfa-a4b3-5d4dc3a89ad3 kind: Event lastTimestamp: "2026-07-15T11:06:59Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:06:59Z" name: ensemble-graph-cf436-7d4dbbfd97-vqcsn.18c271e2480570da namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: cdf4234b-2ba8-4ab7-a2a6-7f2010967be5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cf436} kind: Pod name: ensemble-graph-cf436-7d4dbbfd97-vqcsn namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: bf4f73ea-b479-4bfa-a4b3-5d4dc3a89ad3 kind: Event lastTimestamp: "2026-07-15T11:06:59Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0e1a169e211c4cdcd19b5d9ae2b04ed590c1bcb2f57424f5c6e7d0d0f2faa1f0" already present on machine metadata: creationTimestamp: "2026-07-15T11:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:59Z" name: ensemble-graph-cf436-7d4dbbfd97-vqcsn.18c271e2497e8d43 namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: f6e69657-521d-46f7-a28b-219c295bf3a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cf436} kind: Pod name: ensemble-graph-cf436-7d4dbbfd97-vqcsn namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: bf4f73ea-b479-4bfa-a4b3-5d4dc3a89ad3 kind: Event lastTimestamp: "2026-07-15T11:06:59Z" message: 'Created container: ensemble-graph-cf436' metadata: creationTimestamp: "2026-07-15T11:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:59Z" name: ensemble-graph-cf436-7d4dbbfd97-vqcsn.18c271e24c689c60 namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: 108a6361-72dd-4ddd-961d-9444d42324d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cf436} kind: Pod name: ensemble-graph-cf436-7d4dbbfd97-vqcsn namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: bf4f73ea-b479-4bfa-a4b3-5d4dc3a89ad3 kind: Event lastTimestamp: "2026-07-15T11:06:59Z" message: Started container ensemble-graph-cf436 metadata: creationTimestamp: "2026-07-15T11:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:59Z" name: ensemble-graph-cf436-7d4dbbfd97-vqcsn.18c271e24ce0dbbf namespace: kserve-ci-e2e-test resourceVersion: "20416" uid: e69c4061-e5c9-44a8-869d-f50c889d7439 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cf436} kind: Pod name: ensemble-graph-cf436-7d4dbbfd97-vqcsn namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: bf4f73ea-b479-4bfa-a4b3-5d4dc3a89ad3 kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: Stopping container ensemble-graph-cf436 metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:08Z" name: ensemble-graph-cf436-7d4dbbfd97-vqcsn.18c271e461eaa74b namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: 73b628c6-0e78-4c55-8f69-d734332cf4c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-cf436} kind: Pod name: ensemble-graph-cf436-7d4dbbfd97-vqcsn namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: bf4f73ea-b479-4bfa-a4b3-5d4dc3a89ad3 kind: Event lastTimestamp: "2026-07-15T11:07:35Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-15T11:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:35Z" name: ensemble-graph-cf436-7d4dbbfd97-vqcsn.18c271e4f4b426b5 namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: e0ac11a8-8896-4cdb-9ff7-a32d0ec6ab13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-cf436-7d4dbbfd97 namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: f4e307f8-4e15-4029-9b6c-10eb693308aa kind: Event lastTimestamp: "2026-07-15T11:06:58Z" message: 'Created pod: ensemble-graph-cf436-7d4dbbfd97-vqcsn' metadata: creationTimestamp: "2026-07-15T11:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:58Z" name: ensemble-graph-cf436-7d4dbbfd97.18c271e209527490 namespace: kserve-ci-e2e-test resourceVersion: "20382" uid: 861c0b1c-df75-4fec-931f-79f44adbdeeb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-cf436 namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: 862cb588-6869-46ce-9a6c-0c6d9a8247f1 kind: Event lastTimestamp: "2026-07-15T11:06:58Z" message: Scaled up replica set ensemble-graph-cf436-7d4dbbfd97 from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:58Z" name: ensemble-graph-cf436.18c271e208955d75 namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: 97289eeb-8f3f-414f-a03a-d587d4ca3a06 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:58Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-cf436 namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: 29f3adf0-6b3f-43b2-a6aa-93996c357568 kind: Event lastTimestamp: "2026-07-15T11:06:58Z" message: InferenceGraph [ensemble-graph-cf436] is Ready metadata: creationTimestamp: "2026-07-15T11:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:06:58Z" name: ensemble-graph-cf436.18c271e20f173109 namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: 3ca0251b-89b7-4324-9d98-0c8a304de314 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:08:08.402606Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d6666-6bd466db88-jkkg7 namespace: kserve-ci-e2e-test resourceVersion: "21355" uid: 61f2ac51-b8f5-49f8-8e53-18ac6f78f7d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-d6666-6bd466db88-jkkg7 to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-07-15T11:08:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:08:08Z" name: ensemble-graph-d6666-6bd466db88-jkkg7.18c271f26c264c95 namespace: kserve-ci-e2e-test resourceVersion: "21360" uid: 7b1ed457-f9d4-43d0-91d7-37d39b916768 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:08Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d6666-6bd466db88-jkkg7 namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 61f2ac51-b8f5-49f8-8e53-18ac6f78f7d5 kind: Event lastTimestamp: "2026-07-15T11:08:08Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:08:08Z" name: ensemble-graph-d6666-6bd466db88-jkkg7.18c271f28615856b namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: 3f8fa6a7-9a20-4c76-8613-324938cf1161 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d6666} kind: Pod name: ensemble-graph-d6666-6bd466db88-jkkg7 namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: 61f2ac51-b8f5-49f8-8e53-18ac6f78f7d5 kind: Event lastTimestamp: "2026-07-15T11:08:08Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0e1a169e211c4cdcd19b5d9ae2b04ed590c1bcb2f57424f5c6e7d0d0f2faa1f0" already present on machine metadata: creationTimestamp: "2026-07-15T11:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:08:08Z" name: ensemble-graph-d6666-6bd466db88-jkkg7.18c271f287c35f40 namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: d86ecee9-8c2d-43b1-9764-57d75ccdf5cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d6666} kind: Pod name: ensemble-graph-d6666-6bd466db88-jkkg7 namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: 61f2ac51-b8f5-49f8-8e53-18ac6f78f7d5 kind: Event lastTimestamp: "2026-07-15T11:08:08Z" message: 'Created container: ensemble-graph-d6666' metadata: creationTimestamp: "2026-07-15T11:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:08:08Z" name: ensemble-graph-d6666-6bd466db88-jkkg7.18c271f28aa99116 namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: 27f8b51b-e6df-4346-8f36-afa7cee7414f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d6666} kind: Pod name: ensemble-graph-d6666-6bd466db88-jkkg7 namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: 61f2ac51-b8f5-49f8-8e53-18ac6f78f7d5 kind: Event lastTimestamp: "2026-07-15T11:08:08Z" message: Started container ensemble-graph-d6666 metadata: creationTimestamp: "2026-07-15T11:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:08:08Z" name: ensemble-graph-d6666-6bd466db88-jkkg7.18c271f28b233373 namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: f5a82f33-42b8-41ca-ba36-38aa4072c81a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d6666} kind: Pod name: ensemble-graph-d6666-6bd466db88-jkkg7 namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: 61f2ac51-b8f5-49f8-8e53-18ac6f78f7d5 kind: Event lastTimestamp: "2026-07-15T11:16:22Z" message: Stopping container ensemble-graph-d6666 metadata: creationTimestamp: "2026-07-15T11:16:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:22Z" name: ensemble-graph-d6666-6bd466db88-jkkg7.18c272658fa40647 namespace: kserve-ci-e2e-test resourceVersion: "24875" uid: 93f242d4-08c2-42b4-8394-cf8bbf492add reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d6666} kind: Pod name: ensemble-graph-d6666-6bd466db88-jkkg7 namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: 61f2ac51-b8f5-49f8-8e53-18ac6f78f7d5 kind: Event lastTimestamp: "2026-07-15T11:16:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-15T11:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:50Z" name: ensemble-graph-d6666-6bd466db88-jkkg7.18c272664185cb76 namespace: kserve-ci-e2e-test resourceVersion: "25210" uid: d6d4a581-c8c5-451a-a3e6-95809356a480 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-d6666-6bd466db88 namespace: kserve-ci-e2e-test resourceVersion: "21350" uid: 3f2b62a7-d9f7-4f53-b55b-27e072184188 kind: Event lastTimestamp: "2026-07-15T11:08:08Z" message: 'Created pod: ensemble-graph-d6666-6bd466db88-jkkg7' metadata: creationTimestamp: "2026-07-15T11:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:08Z" name: ensemble-graph-d6666-6bd466db88.18c271f26baa5d3d namespace: kserve-ci-e2e-test resourceVersion: "21356" uid: 381fbd24-03f6-4889-a105-b239dfd3ad0f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-d6666 namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: c9e18f69-4f9b-42c5-b435-ee0dc35d03a6 kind: Event lastTimestamp: "2026-07-15T11:08:08Z" message: Scaled up replica set ensemble-graph-d6666-6bd466db88 from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:08Z" name: ensemble-graph-d6666.18c271f26ac7fcbe namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: cd9457af-2f62-4378-aa23-5602a2b28474 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:08Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-d6666 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 454d6374-8a43-4058-a3bc-77817543042c kind: Event lastTimestamp: "2026-07-15T11:08:08Z" message: InferenceGraph [ensemble-graph-d6666] is Ready metadata: creationTimestamp: "2026-07-15T11:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:08:08Z" name: ensemble-graph-d6666.18c271f270504321 namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: 1ca7ebe1-2b11-4b5a-81a2-61bf916f2757 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:08:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d6666 namespace: kserve-ci-e2e-test resourceVersion: "21349" uid: 23d8b6b7-f18b-447c-b4d9-e77a976df7a8 kind: Event lastTimestamp: "2026-07-15T11:08:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:08:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:38Z" name: ensemble-graph-d6666.18c271f5e937c4a3 namespace: kserve-ci-e2e-test resourceVersion: "21631" uid: c7b3fded-dcdf-4ea7-a330-e32019a4dd4c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:08:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d6666 namespace: kserve-ci-e2e-test resourceVersion: "21349" uid: 23d8b6b7-f18b-447c-b4d9-e77a976df7a8 kind: Event lastTimestamp: "2026-07-15T11:08:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:08:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:38Z" name: ensemble-graph-d6666.18c271f5e99ede32 namespace: kserve-ci-e2e-test resourceVersion: "21632" uid: 36343664-4210-4139-8a1a-75cba5f4beb1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-15T10:57:13.215576Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-074d2-predictor-576646885c-d8cfp namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: bff3e308-3999-4ede-8379-1fdc81f06ae9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-074d2-predictor-576646885c-d8cfp to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T10:57:13Z" name: error-404-isvc-074d2-predictor-576646885c-d8cfp.18c27159dff5a082 namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 40ff43fb-a013-416a-a717-a329cda50df1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-074d2-predictor-576646885c-d8cfp namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: bff3e308-3999-4ede-8379-1fdc81f06ae9 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T10:57:13Z" name: error-404-isvc-074d2-predictor-576646885c-d8cfp.18c27159f9c2b440 namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: abcbe984-67d2-41d7-9f59-eb221825db68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-074d2-predictor-576646885c-d8cfp namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: bff3e308-3999-4ede-8379-1fdc81f06ae9 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: error-404-isvc-074d2-predictor-576646885c-d8cfp.18c27159fb2eb6ac namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: 46b3446f-a32f-4c0a-b88a-1b1b39827db5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-074d2-predictor-576646885c-d8cfp namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: bff3e308-3999-4ede-8379-1fdc81f06ae9 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: error-404-isvc-074d2-predictor-576646885c-d8cfp.18c27159fdb73644 namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: d675f03e-0714-4e1e-bd3d-319e049cc8f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-074d2-predictor-576646885c-d8cfp namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: bff3e308-3999-4ede-8379-1fdc81f06ae9 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: error-404-isvc-074d2-predictor-576646885c-d8cfp.18c27159fe11aa8d namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: e7f3667a-70e9-447b-aa36-17e63485a664 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-074d2-predictor-576646885c-d8cfp namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: bff3e308-3999-4ede-8379-1fdc81f06ae9 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: error-404-isvc-074d2-predictor-576646885c-d8cfp.18c27159fe19f79a namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 2a37e1c9-69c0-4d81-8897-231ddafcd922 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-074d2-predictor-576646885c-d8cfp namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: bff3e308-3999-4ede-8379-1fdc81f06ae9 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: error-404-isvc-074d2-predictor-576646885c-d8cfp.18c2715a01e13c87 namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: b491a9fe-da51-4bdb-90ff-916493fdf064 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-074d2-predictor-576646885c-d8cfp namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: bff3e308-3999-4ede-8379-1fdc81f06ae9 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: error-404-isvc-074d2-predictor-576646885c-d8cfp.18c2715a0247a998 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: a76976cd-464f-4111-9496-b32e862d4b55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-15T10:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-074d2-predictor-576646885c-d8cfp namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: bff3e308-3999-4ede-8379-1fdc81f06ae9 kind: Event lastTimestamp: "2026-07-15T11:06:30Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:30Z" name: error-404-isvc-074d2-predictor-576646885c-d8cfp.18c2715a2b1b5e69 namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 766eea82-2f50-453b-a25d-fbd26ff11b38 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-074d2-predictor-576646885c-d8cfp namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: bff3e308-3999-4ede-8379-1fdc81f06ae9 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-074d2-predictor-576646885c-d8cfp.18c271db23e6b026 namespace: kserve-ci-e2e-test resourceVersion: "20073" uid: ac5c845c-2f6e-42ca-9c00-ff0c467f0945 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-074d2-predictor-576646885c-d8cfp namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: bff3e308-3999-4ede-8379-1fdc81f06ae9 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-074d2-predictor-576646885c-d8cfp.18c271db23e71c76 namespace: kserve-ci-e2e-test resourceVersion: "20074" uid: 476c7b39-3a41-4691-a853-1a99c52349ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-074d2-predictor-576646885c-d8cfp namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: bff3e308-3999-4ede-8379-1fdc81f06ae9 kind: Event lastTimestamp: "2026-07-15T11:06:30Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:30Z" name: error-404-isvc-074d2-predictor-576646885c-d8cfp.18c271db9f92e7c0 namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: 5947194e-7421-4587-b5ad-16a15050565c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-074d2-predictor-576646885c namespace: kserve-ci-e2e-test resourceVersion: "15763" uid: 2787c422-8ba0-4034-9910-4749060bff39 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: 'Created pod: error-404-isvc-074d2-predictor-576646885c-d8cfp' metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:13Z" name: error-404-isvc-074d2-predictor-576646885c.18c27159df8d7c6d namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 1c386906-e74a-4574-a73d-192152dfdeff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-074d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: 18703583-f3a5-4e23-bc5e-f388f94723d1 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Scaled up replica set error-404-isvc-074d2-predictor-576646885c from 0 to 1 metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:13Z" name: error-404-isvc-074d2-predictor.18c27159ddd0a9d4 namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: aaba837c-7dd8-4f41-a4c2-8f0a559ffed0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T10:57:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-074d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: ebd4ec31-0870-479a-8a3b-3820b7f03c4c kind: Event lastTimestamp: "2026-07-15T10:57:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:57:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:43Z" name: error-404-isvc-074d2-predictor.18c2715d5c320292 namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: db5e49b6-fba7-4ace-aa2e-aeb8108238e2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T10:57:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-074d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: ebd4ec31-0870-479a-8a3b-3820b7f03c4c kind: Event lastTimestamp: "2026-07-15T10:57:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:57:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:43Z" name: error-404-isvc-074d2-predictor.18c2715d5cbd6bc6 namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 99ba4dce-24b4-4d8c-bfa5-15c465b3eb7f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:57:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-074d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: ebd4ec31-0870-479a-8a3b-3820b7f03c4c kind: Event lastTimestamp: "2026-07-15T10:58:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:58:28Z" name: error-404-isvc-074d2-predictor.18c271645a55afe7 namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: d6cd5986-d601-4f61-b018-0b9eb1181d23 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:57:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-074d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: ebd4ec31-0870-479a-8a3b-3820b7f03c4c kind: Event lastTimestamp: "2026-07-15T10:58:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:58:28Z" name: error-404-isvc-074d2-predictor.18c271645ac49717 namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: 64c6e4c7-e5d8-4bc4-a541-971d17970fb6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-074d2 namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: 2224b855-1524-42be-9586-3a2b0ba17094 kind: Event lastTimestamp: "2026-07-15T10:58:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:58:00Z" name: error-404-isvc-074d2.18c27159ddd0bf8a namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: a2be04b2-c4fc-44ae-9a36-eb9ea1c66c60 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-074d2 namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: 2224b855-1524-42be-9586-3a2b0ba17094 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: 'Failed to update status for InferenceService "error-404-isvc-074d2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-074d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:57:13Z" name: error-404-isvc-074d2.18c27159de576a38 namespace: kserve-ci-e2e-test resourceVersion: "15768" uid: 84e0dafd-8f53-4930-baf9-ad051c48bf2b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-074d2 namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: 2224b855-1524-42be-9586-3a2b0ba17094 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-074d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:57:13Z" name: error-404-isvc-074d2.18c27159de57ae13 namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: a44befbf-8ea7-4026-8573-b6b1277520e6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:58:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-074d2 namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: 2224b855-1524-42be-9586-3a2b0ba17094 kind: Event lastTimestamp: "2026-07-15T10:58:00Z" message: InferenceService [error-404-isvc-074d2] is Ready metadata: creationTimestamp: "2026-07-15T10:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:58:00Z" name: error-404-isvc-074d2.18c27164e695986e namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 28a25469-a883-442f-ba25-52f2d97f5c43 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:06:28.323426Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6 namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: fa498659-a6ab-4e10-ace6-0400dbd05f68 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-30120-predictor-6c4f57456c-6h8b6 to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6.18c271db1ef73278 namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: 4c7cdd6e-5e26-49bf-8a93-b88258b57abb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6 namespace: kserve-ci-e2e-test resourceVersion: "20066" uid: fa498659-a6ab-4e10-ace6-0400dbd05f68 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6.18c271db39004fcc namespace: kserve-ci-e2e-test resourceVersion: "20084" uid: 618ca9d3-1216-4b9d-84c4-3572f722de3f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: fa498659-a6ab-4e10-ace6-0400dbd05f68 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6.18c271db3ac3c751 namespace: kserve-ci-e2e-test resourceVersion: "20086" uid: 6ebd225b-00f2-45b0-bb98-dac9713ca64c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: fa498659-a6ab-4e10-ace6-0400dbd05f68 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6.18c271db3e51795e namespace: kserve-ci-e2e-test resourceVersion: "20087" uid: be4c23e9-86f5-44b6-a0e2-3815576627b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: fa498659-a6ab-4e10-ace6-0400dbd05f68 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6.18c271db3f07f864 namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: 4840e1a6-5da7-4391-b150-14271e418b6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: fa498659-a6ab-4e10-ace6-0400dbd05f68 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6.18c271db3f12a64f namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: 833df388-c17e-438d-8a06-87d41acbc044 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: fa498659-a6ab-4e10-ace6-0400dbd05f68 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6.18c271db41c25a59 namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 91d2a967-9323-4c17-82b9-c664d78ad860 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: fa498659-a6ab-4e10-ace6-0400dbd05f68 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6.18c271db423314a9 namespace: kserve-ci-e2e-test resourceVersion: "20091" uid: 8a238a66-f28b-4d79-b53c-66b21c06eb38 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-15T11:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: fa498659-a6ab-4e10-ace6-0400dbd05f68 kind: Event lastTimestamp: "2026-07-15T11:07:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:06Z" name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6.18c271db6e507805 namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: c5bd7c50-56ab-4d82-9841-1e709e476a9c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: fa498659-a6ab-4e10-ace6-0400dbd05f68 kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:48Z" name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6.18c271edc1db8826 namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: 4e52f6ac-2ac2-44a2-a875-ae78d5921102 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: fa498659-a6ab-4e10-ace6-0400dbd05f68 kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:48Z" name: error-404-isvc-30120-predictor-6c4f57456c-6h8b6.18c271edc1dc13d4 namespace: kserve-ci-e2e-test resourceVersion: "21088" uid: b6b89939-c7bf-4784-91bd-86238a890f99 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-30120-predictor-6c4f57456c namespace: kserve-ci-e2e-test resourceVersion: "20039" uid: 6633e010-d0dd-408a-b310-cb3e886a3c32 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: 'Created pod: error-404-isvc-30120-predictor-6c4f57456c-6h8b6' metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-30120-predictor-6c4f57456c.18c271db1e42f3d7 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: ad69fdf7-a31b-45fd-ba78-23c39c823938 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-30120-predictor namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: f7470c72-fec0-4105-baa6-e2bf71ef6b20 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: Scaled up replica set error-404-isvc-30120-predictor-6c4f57456c from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-30120-predictor.18c271db1b15d48a namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: 9a9e22b4-8f28-4f2c-82f6-2370b0f9721a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:06:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-30120-predictor namespace: kserve-ci-e2e-test resourceVersion: "20038" uid: a45bd5e5-c716-446b-ad2f-5c4e4a5ab520 kind: Event lastTimestamp: "2026-07-15T11:06:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:58Z" name: error-404-isvc-30120-predictor.18c271de990471cd namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: bfa4dfde-7c44-46f5-b838-7f2c398efe41 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:06:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-30120-predictor namespace: kserve-ci-e2e-test resourceVersion: "20038" uid: a45bd5e5-c716-446b-ad2f-5c4e4a5ab520 kind: Event lastTimestamp: "2026-07-15T11:06:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:58Z" name: error-404-isvc-30120-predictor.18c271de9979cff5 namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: 77f6d8c1-5303-4b2c-865c-1066b2c7bc5e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:07:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-30120-predictor namespace: kserve-ci-e2e-test resourceVersion: "20248" uid: a45bd5e5-c716-446b-ad2f-5c4e4a5ab520 kind: Event lastTimestamp: "2026-07-15T11:07:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:43Z" name: error-404-isvc-30120-predictor.18c271e59779699a namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: b5c82e2c-304e-462b-96ee-87fe285e747d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:07:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-30120-predictor namespace: kserve-ci-e2e-test resourceVersion: "20248" uid: a45bd5e5-c716-446b-ad2f-5c4e4a5ab520 kind: Event lastTimestamp: "2026-07-15T11:07:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:43Z" name: error-404-isvc-30120-predictor.18c271e598c1b69c namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: b172e34e-7e16-42ab-9563-0e48cbe42df6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-30120 namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: 406ba526-0d2f-44e1-a2d2-38424afee073 kind: Event lastTimestamp: "2026-07-15T11:07:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:07:16Z" name: error-404-isvc-30120.18c271db1b1340c8 namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: 14627950-81e6-42b1-b00d-6e0122078b7b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-30120 namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: 406ba526-0d2f-44e1-a2d2-38424afee073 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: 'Failed to update status for InferenceService "error-404-isvc-30120": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-30120": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-30120.18c271db1b9009d7 namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: b576e655-e6e5-4e9c-a634-1ddcaa218b69 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-30120 namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: 406ba526-0d2f-44e1-a2d2-38424afee073 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-30120": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:06:28Z" name: error-404-isvc-30120.18c271db1b905be1 namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: 3709b0ea-6752-43c2-acad-7c6df689c698 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-30120 namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 406ba526-0d2f-44e1-a2d2-38424afee073 kind: Event lastTimestamp: "2026-07-15T11:07:16Z" message: InferenceService [error-404-isvc-30120] is Ready metadata: creationTimestamp: "2026-07-15T11:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:07:16Z" name: error-404-isvc-30120.18c271e665e04d8c namespace: kserve-ci-e2e-test resourceVersion: "20720" uid: 4025bd08-18fc-432f-b688-2ab5cabdb54c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T10:54:43.359469Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14026" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T10:54:43Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c27136fbd6e8c0 namespace: kserve-ci-e2e-test resourceVersion: "14030" uid: c4307061-8c38-4a2d-962c-acc0b942ab7b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6db9e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:43Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c2713708463a24 namespace: kserve-ci-e2e-test resourceVersion: "14055" uid: ffecef5e-6098-4875-9a01-1cd87cb117a3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14035" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:54:44Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T10:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T10:54:44Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c2713740b1d1ea namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 0e0a389a-e479-41e3-b253-262152cebdeb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:54:44Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1738" metadata: creationTimestamp: "2026-07-15T10:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:44Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c2713742756991 namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: 8600f030-545b-4b8a-bb9d-0dead6cb9c83 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1738" in 17.714s (17.714s including waiting). Image size: 1334629472 bytes.' metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c2713b6255cf03 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: f53360ed-c141-422a-8787-8cfc802ad431 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c2713b67676b4a namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: fe8bb1c0-cb79-489c-8358-1a1d3d1b8980 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c2713b68ba56a2 namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: cf87052e-1ffa-4a4a-a25f-1470584ede40 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c2713b68c5f3e8 namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: 7eff4289-945a-43da-a40f-6cc5febf9686 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:55:07Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 5.289s (5.289s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:07Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c2713ca41431a7 namespace: kserve-ci-e2e-test resourceVersion: "14350" uid: 13772e8c-9987-467c-aed7-059b2ec6ca9d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:55:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:07Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c2713cad7f6aa8 namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: ea08f38c-31d1-46ad-b18b-0f5d5560b114 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:55:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:07Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c2713caec88b30 namespace: kserve-ci-e2e-test resourceVersion: "14353" uid: afa0cc08-1b0b-42ae-aa33-6a5f8bc9e40c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c2713cb462cb3f namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 9f6c32af-da93-4b0e-883c-648855c3149f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c2715093b05102 namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: 8ec3caec-33ed-4767-9ac2-962619d052ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c2715093b0bf9c namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: 4b9c891a-3d44-40d9-812c-ce9f2044df8b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 8f43e5ac-68b6-4b9d-af22-40156d0c31da kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g.18c27150baa090fb namespace: kserve-ci-e2e-test resourceVersion: "15278" uid: 55786aa2-9a0f-4e5c-8690-31c08086b55c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6db9e-predictor-6b44c644f9 namespace: kserve-ci-e2e-test resourceVersion: "14013" uid: 17c35f4d-0219-46b1-89af-8df88acb79e4 kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: 'Created pod: error-404-isvc-6db9e-predictor-6b44c644f9-7tc4g' metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:54:43Z" name: error-404-isvc-6db9e-predictor-6b44c644f9.18c27136fb4cddb7 namespace: kserve-ci-e2e-test resourceVersion: "14028" uid: 47b997b4-a8c8-47fc-ab40-6dda87454f52 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6db9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: e02dc162-3bfb-4712-bfbc-9c6e20694e50 kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: Scaled up replica set error-404-isvc-6db9e-predictor-6b44c644f9 from 0 to 1 metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:54:43Z" name: error-404-isvc-6db9e-predictor.18c27136f5298ab6 namespace: kserve-ci-e2e-test resourceVersion: "14017" uid: 199d4e41-931b-48b7-9ef4-afbf123ef0de reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:54:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6db9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 9976b91b-9c53-4444-9188-f832165e3dc2 kind: Event lastTimestamp: "2026-07-15T10:55:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:55:43Z" name: error-404-isvc-6db9e-predictor.18c2713a742e46b4 namespace: kserve-ci-e2e-test resourceVersion: "14666" uid: 31178378-52d9-472d-be99-33bb6f8e8d98 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:54:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6db9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 9976b91b-9c53-4444-9188-f832165e3dc2 kind: Event lastTimestamp: "2026-07-15T10:55:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:55:43Z" name: error-404-isvc-6db9e-predictor.18c2713a7524d0f9 namespace: kserve-ci-e2e-test resourceVersion: "14667" uid: bb6d4c7b-1ba5-419d-9313-1a7bf214a766 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:55:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6db9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 9976b91b-9c53-4444-9188-f832165e3dc2 kind: Event lastTimestamp: "2026-07-15T10:56:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:13Z" name: error-404-isvc-6db9e-predictor.18c2714172b88183 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 77f6405e-7a22-4369-bd34-f8ba8fd0fe52 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:55:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6db9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 9976b91b-9c53-4444-9188-f832165e3dc2 kind: Event lastTimestamp: "2026-07-15T10:56:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:13Z" name: error-404-isvc-6db9e-predictor.18c27141733dda7c namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: cd0da162-7a08-4291-b041-2a5fd1439c5b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6db9e namespace: kserve-ci-e2e-test resourceVersion: "13993" uid: 4a47f570-6f0e-4713-9d12-e2d95e0746f8 kind: Event lastTimestamp: "2026-07-15T10:55:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:55:54Z" name: error-404-isvc-6db9e.18c27136f52c4ee1 namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: fbac8c97-0521-4ea0-b6d3-064f1f45da87 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6db9e namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 4a47f570-6f0e-4713-9d12-e2d95e0746f8 kind: Event lastTimestamp: "2026-07-15T10:55:54Z" message: InferenceService [error-404-isvc-6db9e] is Ready metadata: creationTimestamp: "2026-07-15T10:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:55:54Z" name: error-404-isvc-6db9e.18c2714770705e5f namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: 26a0502e-cc37-4a8f-bf77-0815e3466102 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:07:48.472342Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-70137-predictor-544d7497cf-k6qvv namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 742c19fa-753e-49c9-9b71-ca36ef4dd9bc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-70137-predictor-544d7497cf-k6qvv to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:07:48Z" name: error-404-isvc-70137-predictor-544d7497cf-k6qvv.18c271edc8369d4f namespace: kserve-ci-e2e-test resourceVersion: "21112" uid: ca322343-dc4c-436d-8fd1-5afa662890ff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-70137-predictor-544d7497cf-k6qvv namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: 742c19fa-753e-49c9-9b71-ca36ef4dd9bc kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:07:48Z" name: error-404-isvc-70137-predictor-544d7497cf-k6qvv.18c271ede2195bff namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: 2171f3f0-f262-483a-b526-d4a146d09fb3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-70137-predictor-544d7497cf-k6qvv namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: 742c19fa-753e-49c9-9b71-ca36ef4dd9bc kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:48Z" name: error-404-isvc-70137-predictor-544d7497cf-k6qvv.18c271ede3676935 namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 072aaebd-40e1-45a6-a9af-9ee00ad86ff3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-70137-predictor-544d7497cf-k6qvv namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: 742c19fa-753e-49c9-9b71-ca36ef4dd9bc kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:48Z" name: error-404-isvc-70137-predictor-544d7497cf-k6qvv.18c271ede6a6e6b0 namespace: kserve-ci-e2e-test resourceVersion: "21133" uid: 948f93d5-4843-4f44-a2ca-d82a2e76a651 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-70137-predictor-544d7497cf-k6qvv namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: 742c19fa-753e-49c9-9b71-ca36ef4dd9bc kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:48Z" name: error-404-isvc-70137-predictor-544d7497cf-k6qvv.18c271ede71925e7 namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: 892a22ad-4ad5-445d-a65d-5c15d99b893b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-70137-predictor-544d7497cf-k6qvv namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: 742c19fa-753e-49c9-9b71-ca36ef4dd9bc kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:48Z" name: error-404-isvc-70137-predictor-544d7497cf-k6qvv.18c271ede724aa5b namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: 222db525-4423-40c6-88e3-e95093d81503 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-70137-predictor-544d7497cf-k6qvv namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: 742c19fa-753e-49c9-9b71-ca36ef4dd9bc kind: Event lastTimestamp: "2026-07-15T11:07:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:49Z" name: error-404-isvc-70137-predictor-544d7497cf-k6qvv.18c271edea648593 namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: 67bbbcfa-b57d-4ba7-95e1-91fbafc8d9c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-70137-predictor-544d7497cf-k6qvv namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: 742c19fa-753e-49c9-9b71-ca36ef4dd9bc kind: Event lastTimestamp: "2026-07-15T11:07:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:49Z" name: error-404-isvc-70137-predictor-544d7497cf-k6qvv.18c271edead5b12a namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: 50ccbb10-0a08-4c10-bac6-48676a9da1de reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-15T11:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-70137-predictor-544d7497cf-k6qvv namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: 742c19fa-753e-49c9-9b71-ca36ef4dd9bc kind: Event lastTimestamp: "2026-07-15T11:17:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:07Z" name: error-404-isvc-70137-predictor-544d7497cf-k6qvv.18c271ee5feb9c76 namespace: kserve-ci-e2e-test resourceVersion: "25476" uid: 0b346278-1b25-43f1-ae15-dce421f2b58e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-70137-predictor-544d7497cf-k6qvv namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: 742c19fa-753e-49c9-9b71-ca36ef4dd9bc kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-70137-predictor-544d7497cf-k6qvv.18c2726eec9e3865 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: fe14111f-555c-4712-8cac-ae573ba5b170 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-70137-predictor-544d7497cf-k6qvv namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: 742c19fa-753e-49c9-9b71-ca36ef4dd9bc kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-70137-predictor-544d7497cf-k6qvv.18c2726eec9f7739 namespace: kserve-ci-e2e-test resourceVersion: "25342" uid: d966761d-e3ae-497a-9945-8a1575c403ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-70137-predictor-544d7497cf-k6qvv namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: 742c19fa-753e-49c9-9b71-ca36ef4dd9bc kind: Event lastTimestamp: "2026-07-15T11:17:07Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:07Z" name: error-404-isvc-70137-predictor-544d7497cf-k6qvv.18c2726fd48c0e8b namespace: kserve-ci-e2e-test resourceVersion: "25475" uid: 717aa70b-dba8-4ce7-aeb1-625d1ee5378a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-70137-predictor-544d7497cf namespace: kserve-ci-e2e-test resourceVersion: "21101" uid: b22f32af-d8a0-49e1-9623-34c35fafc872 kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: 'Created pod: error-404-isvc-70137-predictor-544d7497cf-k6qvv' metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:48Z" name: error-404-isvc-70137-predictor-544d7497cf.18c271edc7bf8198 namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: 362f459c-6aa0-4d15-a4cb-118732d7b268 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-70137-predictor namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 0860d48a-db05-4eb7-8c41-2252d272943f kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: Scaled up replica set error-404-isvc-70137-predictor-544d7497cf from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:48Z" name: error-404-isvc-70137-predictor.18c271edc5a5d83e namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: 3d807b2a-fabe-4153-ad5d-996e8acfe0b5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-70137-predictor namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 7ef50e1d-d034-44db-a164-9e890e2b655d kind: Event lastTimestamp: "2026-07-15T11:08:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:18Z" name: error-404-isvc-70137-predictor.18c271f144280922 namespace: kserve-ci-e2e-test resourceVersion: "21459" uid: cbdb59fa-c51a-40fe-bb87-f277f3690d26 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-70137-predictor namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 7ef50e1d-d034-44db-a164-9e890e2b655d kind: Event lastTimestamp: "2026-07-15T11:08:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:18Z" name: error-404-isvc-70137-predictor.18c271f1449729c0 namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 9f286ab7-a9a2-409b-866a-c3e5cc38036b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-15T11:08:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-70137-predictor namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 7ef50e1d-d034-44db-a164-9e890e2b655d kind: Event lastTimestamp: "2026-07-15T11:09:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:08:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:09:18Z" name: error-404-isvc-70137-predictor.18c271f842134bb5 namespace: kserve-ci-e2e-test resourceVersion: "21968" uid: e361ad99-a142-493c-9365-d08c1dc9a889 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-15T11:08:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-70137-predictor namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 7ef50e1d-d034-44db-a164-9e890e2b655d kind: Event lastTimestamp: "2026-07-15T11:09:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:08:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:09:18Z" name: error-404-isvc-70137-predictor.18c271f84272920a namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: 4178eb96-b73e-44bf-9c6d-b7a948752fac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-70137 namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: 028ef890-8959-4a7a-bde8-c4a937ef180c kind: Event lastTimestamp: "2026-07-15T11:08:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:08:37Z" name: error-404-isvc-70137.18c271edc5b30bbb namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 4eb61c24-e7c3-4bb3-9bb4-86f0aaa843b7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-70137 namespace: kserve-ci-e2e-test resourceVersion: "21616" uid: 028ef890-8959-4a7a-bde8-c4a937ef180c kind: Event lastTimestamp: "2026-07-15T11:08:37Z" message: InferenceService [error-404-isvc-70137] is Ready metadata: creationTimestamp: "2026-07-15T11:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:08:37Z" name: error-404-isvc-70137.18c271f91ed4c325 namespace: kserve-ci-e2e-test resourceVersion: "21618" uid: cee4fc29-e010-4082-a4c3-23f75f3ea071 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:17:03.348017Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb namespace: kserve-ci-e2e-test resourceVersion: "25406" uid: e4a430db-cad0-4b4b-b3a1-ba9a562b6dc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bc774-predictor-8547545bdf-ktmgb to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb.18c2726ef9617f09 namespace: kserve-ci-e2e-test resourceVersion: "25411" uid: 51bccc34-0871-488a-b660-210fb8b62c20 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb namespace: kserve-ci-e2e-test resourceVersion: "25417" uid: e4a430db-cad0-4b4b-b3a1-ba9a562b6dc7 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb.18c2726f144ed278 namespace: kserve-ci-e2e-test resourceVersion: "25433" uid: 5dbe8fcf-5ae5-49c2-a614-2a056b361709 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb namespace: kserve-ci-e2e-test resourceVersion: "25408" uid: e4a430db-cad0-4b4b-b3a1-ba9a562b6dc7 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb.18c2726f15c17260 namespace: kserve-ci-e2e-test resourceVersion: "25435" uid: 952f244a-d7b6-43b2-98c8-8e98e444a49d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb namespace: kserve-ci-e2e-test resourceVersion: "25408" uid: e4a430db-cad0-4b4b-b3a1-ba9a562b6dc7 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb.18c2726f18255c59 namespace: kserve-ci-e2e-test resourceVersion: "25438" uid: df661916-96ae-4668-8327-ce29256220ba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb namespace: kserve-ci-e2e-test resourceVersion: "25408" uid: e4a430db-cad0-4b4b-b3a1-ba9a562b6dc7 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb.18c2726f18990ca2 namespace: kserve-ci-e2e-test resourceVersion: "25439" uid: ffc615ec-8702-455e-ae0c-295f54d4eef2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb namespace: kserve-ci-e2e-test resourceVersion: "25408" uid: e4a430db-cad0-4b4b-b3a1-ba9a562b6dc7 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb.18c2726f18a2deb6 namespace: kserve-ci-e2e-test resourceVersion: "25440" uid: 680ffb7c-b1b9-4c32-832f-f3add04027ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb namespace: kserve-ci-e2e-test resourceVersion: "25408" uid: e4a430db-cad0-4b4b-b3a1-ba9a562b6dc7 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb.18c2726f1b20f894 namespace: kserve-ci-e2e-test resourceVersion: "25441" uid: 1dd3ba7b-b62d-4697-8d2f-c26eb7a6efdc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb namespace: kserve-ci-e2e-test resourceVersion: "25408" uid: e4a430db-cad0-4b4b-b3a1-ba9a562b6dc7 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb.18c2726f1b95e6d6 namespace: kserve-ci-e2e-test resourceVersion: "25442" uid: 58930e32-fe38-4e04-8c22-7cdb238ee2f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-15T11:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb namespace: kserve-ci-e2e-test resourceVersion: "25408" uid: e4a430db-cad0-4b4b-b3a1-ba9a562b6dc7 kind: Event lastTimestamp: "2026-07-15T11:17:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:51Z" name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb.18c2726f2abb3308 namespace: kserve-ci-e2e-test resourceVersion: "26349" uid: ec86834e-2b93-4157-8e71-398b03433d40 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb namespace: kserve-ci-e2e-test resourceVersion: "25408" uid: e4a430db-cad0-4b4b-b3a1-ba9a562b6dc7 kind: Event lastTimestamp: "2026-07-15T11:34:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:34:22Z" name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb.18c27360f1b37834 namespace: kserve-ci-e2e-test resourceVersion: "33382" uid: 36dd0a49-559e-488d-977d-41e622f63618 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb namespace: kserve-ci-e2e-test resourceVersion: "25408" uid: e4a430db-cad0-4b4b-b3a1-ba9a562b6dc7 kind: Event lastTimestamp: "2026-07-15T11:34:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:34:22Z" name: error-404-isvc-bc774-predictor-8547545bdf-ktmgb.18c27360f1b3f73a namespace: kserve-ci-e2e-test resourceVersion: "33383" uid: 3312676f-a729-40a8-9b08-ff972c5abb5a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bc774-predictor-8547545bdf namespace: kserve-ci-e2e-test resourceVersion: "25398" uid: 9b0efc96-ff52-4d2c-a4db-2ba2115be852 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: 'Created pod: error-404-isvc-bc774-predictor-8547545bdf-ktmgb' metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-bc774-predictor-8547545bdf.18c2726ef8e004a5 namespace: kserve-ci-e2e-test resourceVersion: "25407" uid: 7702a480-d190-4d5f-a42c-1aedf4766899 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bc774-predictor namespace: kserve-ci-e2e-test resourceVersion: "25396" uid: 3b87be1c-339c-426e-b5df-8c335804b8f5 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Scaled up replica set error-404-isvc-bc774-predictor-8547545bdf from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-bc774-predictor.18c2726ef7533f30 namespace: kserve-ci-e2e-test resourceVersion: "25400" uid: 78713e65-5f46-4770-a87b-c96bede81b43 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:17:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bc774-predictor namespace: kserve-ci-e2e-test resourceVersion: "25397" uid: 5984f5d4-1f74-4d1f-b7e5-a0a781a8c701 kind: Event lastTimestamp: "2026-07-15T11:17:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:48Z" name: error-404-isvc-bc774-predictor.18c2727275e732e8 namespace: kserve-ci-e2e-test resourceVersion: "26321" uid: 9a01d1ae-648e-4727-a9ef-0c5e2645672a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:17:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bc774-predictor namespace: kserve-ci-e2e-test resourceVersion: "25397" uid: 5984f5d4-1f74-4d1f-b7e5-a0a781a8c701 kind: Event lastTimestamp: "2026-07-15T11:17:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:48Z" name: error-404-isvc-bc774-predictor.18c27272765ce0e7 namespace: kserve-ci-e2e-test resourceVersion: "26323" uid: e6319ea2-4c5d-460e-982d-62875ebd2b9f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bc774-predictor namespace: kserve-ci-e2e-test resourceVersion: "25598" uid: 5984f5d4-1f74-4d1f-b7e5-a0a781a8c701 kind: Event lastTimestamp: "2026-07-15T11:18:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:18Z" name: error-404-isvc-bc774-predictor.18c27275f501a2b9 namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: ebd97c95-e48f-4578-905c-651095653903 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bc774-predictor namespace: kserve-ci-e2e-test resourceVersion: "25598" uid: 5984f5d4-1f74-4d1f-b7e5-a0a781a8c701 kind: Event lastTimestamp: "2026-07-15T11:18:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:18Z" name: error-404-isvc-bc774-predictor.18c27275f5974de5 namespace: kserve-ci-e2e-test resourceVersion: "26623" uid: 034b1baa-b894-4d22-bbbe-b3dce0fe851f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bc774 namespace: kserve-ci-e2e-test resourceVersion: "25386" uid: a00acc13-8bce-4bea-a178-e5c8ffb54403 kind: Event lastTimestamp: "2026-07-15T11:18:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:18:01Z" name: error-404-isvc-bc774.18c2726ef751db4a namespace: kserve-ci-e2e-test resourceVersion: "26442" uid: 4d6bb912-621d-462f-80b0-d1d9a1a1f6f2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bc774 namespace: kserve-ci-e2e-test resourceVersion: "25386" uid: a00acc13-8bce-4bea-a178-e5c8ffb54403 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: 'Failed to update status for InferenceService "error-404-isvc-bc774": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bc774": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-bc774.18c2726ef7bdcc10 namespace: kserve-ci-e2e-test resourceVersion: "25402" uid: 003e96d5-d5b5-425c-84da-c69e654176cd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bc774 namespace: kserve-ci-e2e-test resourceVersion: "25386" uid: a00acc13-8bce-4bea-a178-e5c8ffb54403 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bc774": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:03Z" name: error-404-isvc-bc774.18c2726ef7be0d00 namespace: kserve-ci-e2e-test resourceVersion: "25403" uid: 373cbf9d-6571-41a9-a42c-89bef3616186 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bc774 namespace: kserve-ci-e2e-test resourceVersion: "26440" uid: a00acc13-8bce-4bea-a178-e5c8ffb54403 kind: Event lastTimestamp: "2026-07-15T11:18:01Z" message: InferenceService [error-404-isvc-bc774] is Ready metadata: creationTimestamp: "2026-07-15T11:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:18:01Z" name: error-404-isvc-bc774.18c2727c76296b50 namespace: kserve-ci-e2e-test resourceVersion: "26441" uid: 737859ac-0a8c-45d1-a7cc-262a824c0c63 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:16:23.188873Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs namespace: kserve-ci-e2e-test resourceVersion: "24955" uid: 148f5d1c-e482-441e-8420-cdc14896673b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-cdb7a-predictor-66db674dcc-48hvs to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs.18c272659fb59589 namespace: kserve-ci-e2e-test resourceVersion: "24962" uid: d1801294-bcb7-43e7-89e8-862a1f40c095 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: 148f5d1c-e482-441e-8420-cdc14896673b kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs.18c27265ba2ddd23 namespace: kserve-ci-e2e-test resourceVersion: "24982" uid: 8edac1aa-546c-47ea-aac0-7c225e586826 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 148f5d1c-e482-441e-8420-cdc14896673b kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs.18c27265bbcfae1c namespace: kserve-ci-e2e-test resourceVersion: "24984" uid: e4f96b19-79d5-4b12-bf67-7bede8ed228b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 148f5d1c-e482-441e-8420-cdc14896673b kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs.18c27265bee485af namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 46345315-cdeb-420e-9c5f-55203f85433f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 148f5d1c-e482-441e-8420-cdc14896673b kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs.18c27265bf5e4661 namespace: kserve-ci-e2e-test resourceVersion: "24990" uid: 592abade-e8f3-4d0c-91ba-34d3202d5b99 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 148f5d1c-e482-441e-8420-cdc14896673b kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs.18c27265bf697660 namespace: kserve-ci-e2e-test resourceVersion: "24991" uid: 6b9d1979-f837-4e09-97bb-9f96a9aaa889 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 148f5d1c-e482-441e-8420-cdc14896673b kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs.18c27265c34021a2 namespace: kserve-ci-e2e-test resourceVersion: "24993" uid: 3beec881-e5e9-46e4-a50f-88991e7bf9f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 148f5d1c-e482-441e-8420-cdc14896673b kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs.18c27265c3bee33a namespace: kserve-ci-e2e-test resourceVersion: "24994" uid: ca827ac5-b82c-4ce9-9997-720688c46907 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 148f5d1c-e482-441e-8420-cdc14896673b kind: Event lastTimestamp: "2026-07-15T11:17:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:01Z" name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs.18c272660be633c4 namespace: kserve-ci-e2e-test resourceVersion: "25290" uid: 2d71c4d9-6876-46a7-b264-3ca7db46bd77 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 148f5d1c-e482-441e-8420-cdc14896673b kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:33Z" name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs.18c27275f6e7280f namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 23d193e0-7299-4e63-98f2-3a62a5dea23a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 148f5d1c-e482-441e-8420-cdc14896673b kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:33Z" name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs.18c27275f6e7a47e namespace: kserve-ci-e2e-test resourceVersion: "26090" uid: d8c87fba-070d-47d0-8904-14d34e063023 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 148f5d1c-e482-441e-8420-cdc14896673b kind: Event lastTimestamp: "2026-07-15T11:17:36Z" message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp 10.133.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:17:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:36Z" name: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs.18c27276946566eb namespace: kserve-ci-e2e-test resourceVersion: "26213" uid: ebdd75c1-3b91-4492-b61c-2b62c0c016eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-cdb7a-predictor-66db674dcc namespace: kserve-ci-e2e-test resourceVersion: "24940" uid: 1838e0c0-366f-4f86-bf5b-ffc414e2ff67 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: 'Created pod: error-404-isvc-cdb7a-predictor-66db674dcc-48hvs' metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-cdb7a-predictor-66db674dcc.18c272659f2c1f7d namespace: kserve-ci-e2e-test resourceVersion: "24957" uid: 10eada74-4de3-4dc5-b4ac-360a1d07f227 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-cdb7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24931" uid: 8073bea9-7729-4c22-b8c4-f4fc8e93aa90 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Scaled up replica set error-404-isvc-cdb7a-predictor-66db674dcc from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-cdb7a-predictor.18c272659d79d3c9 namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: 4991c714-aaa5-477a-ae12-d80b35cfb886 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:16:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cdb7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24938" uid: b23d78ed-44eb-48e0-ac16-668be49787a2 kind: Event lastTimestamp: "2026-07-15T11:17:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:16:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:23Z" name: error-404-isvc-cdb7a-predictor.18c272691c325f32 namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: 576cc61f-2b42-4a06-a0d2-1db58a6b2524 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:16:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cdb7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24938" uid: b23d78ed-44eb-48e0-ac16-668be49787a2 kind: Event lastTimestamp: "2026-07-15T11:17:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:16:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:23Z" name: error-404-isvc-cdb7a-predictor.18c272691c8e9180 namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: 99a251fd-c36d-4bfc-88e6-b99ab901d3b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cdb7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25122" uid: b23d78ed-44eb-48e0-ac16-668be49787a2 kind: Event lastTimestamp: "2026-07-15T11:16:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:16:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:16:53Z" name: error-404-isvc-cdb7a-predictor.18c2726c9b4ba747 namespace: kserve-ci-e2e-test resourceVersion: "25231" uid: 810c9295-5ae9-4490-a7d9-5cdc4c2a23f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cdb7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25122" uid: b23d78ed-44eb-48e0-ac16-668be49787a2 kind: Event lastTimestamp: "2026-07-15T11:16:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:16:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:16:53Z" name: error-404-isvc-cdb7a-predictor.18c2726c9bbb6a9f namespace: kserve-ci-e2e-test resourceVersion: "25233" uid: 43615714-2e00-4482-ad53-edaadf125754 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cdb7a namespace: kserve-ci-e2e-test resourceVersion: "24913" uid: 890e41ac-b41d-4b93-9ae1-84b7bffbf51e kind: Event lastTimestamp: "2026-07-15T11:17:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:11Z" name: error-404-isvc-cdb7a.18c272659d8cf844 namespace: kserve-ci-e2e-test resourceVersion: "25537" uid: 0eb0d01c-2c2f-4696-b9f3-248b340ad439 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cdb7a namespace: kserve-ci-e2e-test resourceVersion: "24913" uid: 890e41ac-b41d-4b93-9ae1-84b7bffbf51e kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: 'Failed to update status for InferenceService "error-404-isvc-cdb7a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cdb7a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-cdb7a.18c272659e20228d namespace: kserve-ci-e2e-test resourceVersion: "24951" uid: 30650f0d-ebb9-48b7-9cc4-2a5c2702c931 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cdb7a namespace: kserve-ci-e2e-test resourceVersion: "24913" uid: 890e41ac-b41d-4b93-9ae1-84b7bffbf51e kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cdb7a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-cdb7a.18c272659e207742 namespace: kserve-ci-e2e-test resourceVersion: "24954" uid: 3ffc0816-b53f-4e58-8693-a1439d6cde23 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cdb7a namespace: kserve-ci-e2e-test resourceVersion: "25531" uid: 890e41ac-b41d-4b93-9ae1-84b7bffbf51e kind: Event lastTimestamp: "2026-07-15T11:17:11Z" message: InferenceService [error-404-isvc-cdb7a] is Ready metadata: creationTimestamp: "2026-07-15T11:17:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:11Z" name: error-404-isvc-cdb7a.18c27270c8057960 namespace: kserve-ci-e2e-test resourceVersion: "25532" uid: 9cc988b3-31dc-4927-92d6-98405293c312 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:05:48.053588Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6 namespace: kserve-ci-e2e-test resourceVersion: "19619" uid: f62dc8da-e540-4422-af99-61e6d8791537 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-cf436-predictor-777f9bff5f-j9df6 to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6.18c271d1beb23e22 namespace: kserve-ci-e2e-test resourceVersion: "19624" uid: e84825a4-e58f-409b-9445-5a4f1f5ecebf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6 namespace: kserve-ci-e2e-test resourceVersion: "19629" uid: f62dc8da-e540-4422-af99-61e6d8791537 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6.18c271d1d8c86951 namespace: kserve-ci-e2e-test resourceVersion: "19642" uid: 1226feb6-f506-48c4-92c1-5633704e5e13 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6 namespace: kserve-ci-e2e-test resourceVersion: "19622" uid: f62dc8da-e540-4422-af99-61e6d8791537 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6.18c271d1da6cae15 namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: 1a8330af-6d9e-408b-a76d-e613a629754b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6 namespace: kserve-ci-e2e-test resourceVersion: "19622" uid: f62dc8da-e540-4422-af99-61e6d8791537 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6.18c271d1dce6e1c3 namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: dfe77250-0a5a-4d2a-9914-510331bca4e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6 namespace: kserve-ci-e2e-test resourceVersion: "19622" uid: f62dc8da-e540-4422-af99-61e6d8791537 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6.18c271d1dd547883 namespace: kserve-ci-e2e-test resourceVersion: "19652" uid: ae7383f2-59bc-4ea1-8dce-ca85e21a1b2b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6 namespace: kserve-ci-e2e-test resourceVersion: "19622" uid: f62dc8da-e540-4422-af99-61e6d8791537 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6.18c271d1dd5d6b24 namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: 3b6d49d8-1d9d-4c2f-aacc-67fc7e894e1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6 namespace: kserve-ci-e2e-test resourceVersion: "19622" uid: f62dc8da-e540-4422-af99-61e6d8791537 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6.18c271d1dff35d90 namespace: kserve-ci-e2e-test resourceVersion: "19654" uid: 97a629f9-ee6e-40c4-8905-4fb7d4dfaa18 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6 namespace: kserve-ci-e2e-test resourceVersion: "19622" uid: f62dc8da-e540-4422-af99-61e6d8791537 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6.18c271d1e07fe234 namespace: kserve-ci-e2e-test resourceVersion: "19655" uid: 2e19c9e7-a241-4375-9ba5-f5275b66b679 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:05:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6 namespace: kserve-ci-e2e-test resourceVersion: "19622" uid: f62dc8da-e540-4422-af99-61e6d8791537 kind: Event lastTimestamp: "2026-07-15T11:06:25Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:25Z" name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6.18c271d213c51e1a namespace: kserve-ci-e2e-test resourceVersion: "19949" uid: 58ed9121-b71b-4815-bce9-b8c4d9bd6b9c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6 namespace: kserve-ci-e2e-test resourceVersion: "19622" uid: f62dc8da-e540-4422-af99-61e6d8791537 kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:08Z" name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6.18c271e467f2bcbb namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: f353b6ea-df70-4f7a-8a21-132ac443193f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6 namespace: kserve-ci-e2e-test resourceVersion: "19622" uid: f62dc8da-e540-4422-af99-61e6d8791537 kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:08Z" name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6.18c271e467f33bd2 namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: e2d70440-2027-44da-b276-01dd30aa3b03 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6 namespace: kserve-ci-e2e-test resourceVersion: "19622" uid: f62dc8da-e540-4422-af99-61e6d8791537 kind: Event lastTimestamp: "2026-07-15T11:07:10Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:10Z" name: error-404-isvc-cf436-predictor-777f9bff5f-j9df6.18c271e4effaf689 namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: 1826a24d-f86a-4c4b-a277-7aeba38f742c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-cf436-predictor-777f9bff5f namespace: kserve-ci-e2e-test resourceVersion: "19609" uid: 49e740a2-18d8-455d-965e-328bf3f8dbdb kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: 'Created pod: error-404-isvc-cf436-predictor-777f9bff5f-j9df6' metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-cf436-predictor-777f9bff5f.18c271d1be33fcff namespace: kserve-ci-e2e-test resourceVersion: "19621" uid: e5d6c3d6-55d0-4b60-8693-804b8882d38b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-cf436-predictor namespace: kserve-ci-e2e-test resourceVersion: "19604" uid: 072720c7-ba88-438c-a712-70ffaec7f9b7 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: Scaled up replica set error-404-isvc-cf436-predictor-777f9bff5f from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-cf436-predictor.18c271d1bc851110 namespace: kserve-ci-e2e-test resourceVersion: "19611" uid: 638d1f52-a2e0-4ff2-b948-a5a80ee855a8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:06:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cf436-predictor namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: 3a757b6a-7817-4175-becb-22c91e407d6b kind: Event lastTimestamp: "2026-07-15T11:06:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:18Z" name: error-404-isvc-cf436-predictor.18c271d53b161d85 namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: 581da6df-90fb-4b50-b0b3-20b8e3c5e2cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:06:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cf436-predictor namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: 3a757b6a-7817-4175-becb-22c91e407d6b kind: Event lastTimestamp: "2026-07-15T11:06:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:18Z" name: error-404-isvc-cf436-predictor.18c271d53b93bda3 namespace: kserve-ci-e2e-test resourceVersion: "19900" uid: de570b4a-cf3b-4fa4-9379-cf966344bf13 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:06:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cf436-predictor namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 3a757b6a-7817-4175-becb-22c91e407d6b kind: Event lastTimestamp: "2026-07-15T11:07:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:03Z" name: error-404-isvc-cf436-predictor.18c271dc39be8383 namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 80c3e109-83d5-4efe-a7b7-3eb7ca1cdd41 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:06:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cf436-predictor namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 3a757b6a-7817-4175-becb-22c91e407d6b kind: Event lastTimestamp: "2026-07-15T11:07:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:06:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:03Z" name: error-404-isvc-cf436-predictor.18c271dc3a61689c namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: c10876ed-05c4-4d7e-bf21-43e353976a5f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cf436 namespace: kserve-ci-e2e-test resourceVersion: "19577" uid: dc63ead0-af12-4211-83f5-a6e05ec923bb kind: Event lastTimestamp: "2026-07-15T11:06:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:06:35Z" name: error-404-isvc-cf436.18c271d1bc8789e5 namespace: kserve-ci-e2e-test resourceVersion: "20182" uid: 27fc5eaa-8ea4-4e02-ae18-b335e408d06a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cf436 namespace: kserve-ci-e2e-test resourceVersion: "19577" uid: dc63ead0-af12-4211-83f5-a6e05ec923bb kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: 'Failed to update status for InferenceService "error-404-isvc-cf436": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cf436": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-cf436.18c271d1bd13afa8 namespace: kserve-ci-e2e-test resourceVersion: "19616" uid: c0bfa4c0-575c-4ec6-8cf3-48806c5931b0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cf436 namespace: kserve-ci-e2e-test resourceVersion: "19577" uid: dc63ead0-af12-4211-83f5-a6e05ec923bb kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cf436": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-cf436.18c271d1bd140255 namespace: kserve-ci-e2e-test resourceVersion: "19617" uid: 7f10cd89-7f7a-41d6-9aa1-81866230c71b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cf436 namespace: kserve-ci-e2e-test resourceVersion: "20180" uid: dc63ead0-af12-4211-83f5-a6e05ec923bb kind: Event lastTimestamp: "2026-07-15T11:06:35Z" message: InferenceService [error-404-isvc-cf436] is Ready metadata: creationTimestamp: "2026-07-15T11:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:06:35Z" name: error-404-isvc-cf436.18c271dccfb74875 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 56f3714c-d02e-42b1-b2fb-11dbdd62488b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:17:33.605252Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 namespace: kserve-ci-e2e-test resourceVersion: "26153" uid: b0830e31-83c6-47ee-993b-9ee52899112f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:17:33Z" name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2.18c2727604da4e5f namespace: kserve-ci-e2e-test resourceVersion: "26157" uid: 59025349-fbba-4dad-9a85-6a50a913a22e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: b0830e31-83c6-47ee-993b-9ee52899112f kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d1ffd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:33Z" name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2.18c272760d9a90a7 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: b867ebc6-06d5-4cca-8c93-356d539a0e70 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 namespace: kserve-ci-e2e-test resourceVersion: "26165" uid: b0830e31-83c6-47ee-993b-9ee52899112f kind: Event lastTimestamp: "2026-07-15T11:17:34Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:17:34Z" name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2.18c27276423e42b5 namespace: kserve-ci-e2e-test resourceVersion: "26191" uid: 7faf0d14-2e23-45ae-a641-9a0373cfcaf8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: b0830e31-83c6-47ee-993b-9ee52899112f kind: Event lastTimestamp: "2026-07-15T11:17:34Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:34Z" name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2.18c2727643d19264 namespace: kserve-ci-e2e-test resourceVersion: "26193" uid: c17cfdf8-08bb-4dfc-bcfb-79a94c4cc4f5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: b0830e31-83c6-47ee-993b-9ee52899112f kind: Event lastTimestamp: "2026-07-15T11:17:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:34Z" name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2.18c2727646f3fd75 namespace: kserve-ci-e2e-test resourceVersion: "26194" uid: c866067c-8aef-43ea-9f23-b104032e60a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: b0830e31-83c6-47ee-993b-9ee52899112f kind: Event lastTimestamp: "2026-07-15T11:17:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:34Z" name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2.18c2727647675138 namespace: kserve-ci-e2e-test resourceVersion: "26195" uid: 6af0f34f-1dfe-4e81-8752-ff11493830d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: b0830e31-83c6-47ee-993b-9ee52899112f kind: Event lastTimestamp: "2026-07-15T11:17:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:34Z" name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2.18c272764771f4f8 namespace: kserve-ci-e2e-test resourceVersion: "26196" uid: c8ed5b65-6d46-467a-940d-86a618aca43b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: b0830e31-83c6-47ee-993b-9ee52899112f kind: Event lastTimestamp: "2026-07-15T11:17:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:34Z" name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2.18c272764a8645fa namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 49e0cb82-8a41-4278-97f1-33a5edecc385 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: b0830e31-83c6-47ee-993b-9ee52899112f kind: Event lastTimestamp: "2026-07-15T11:17:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:34Z" name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2.18c272764b173f34 namespace: kserve-ci-e2e-test resourceVersion: "26199" uid: 0ceed21d-f810-42b3-97d4-264bedffd644 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-15T11:17:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: b0830e31-83c6-47ee-993b-9ee52899112f kind: Event lastTimestamp: "2026-07-15T11:26:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:26:51Z" name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2.18c272766ba6716f namespace: kserve-ci-e2e-test resourceVersion: "30262" uid: c43d2889-14f1-4e09-90a4-316628b36fa5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:26:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: b0830e31-83c6-47ee-993b-9ee52899112f kind: Event lastTimestamp: "2026-07-15T11:26:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:26:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:26:48Z" name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2.18c272f7409cecc7 namespace: kserve-ci-e2e-test resourceVersion: "30237" uid: 9656e326-b141-4fdb-b8d4-476c6853dbc3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:26:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: b0830e31-83c6-47ee-993b-9ee52899112f kind: Event lastTimestamp: "2026-07-15T11:26:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:26:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:26:48Z" name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2.18c272f7409da68f namespace: kserve-ci-e2e-test resourceVersion: "30238" uid: 1cddd0ac-8c9f-4ac0-b44f-d70be5037b93 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: b0830e31-83c6-47ee-993b-9ee52899112f kind: Event lastTimestamp: "2026-07-15T11:26:51Z" message: 'Readiness probe failed: Get "https://10.133.0.43:8643/healthz": dial tcp 10.133.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:26:51Z" name: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2.18c272f7e03f5706 namespace: kserve-ci-e2e-test resourceVersion: "30261" uid: f1a4c867-1c46-4d59-b243-d4fbf0c965c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d1ffd-predictor-bb8d46565 namespace: kserve-ci-e2e-test resourceVersion: "26144" uid: 3a3a65b5-5556-4ee2-b1ae-fd205a541023 kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: 'Created pod: error-404-isvc-d1ffd-predictor-bb8d46565-r7hp2' metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:33Z" name: error-404-isvc-d1ffd-predictor-bb8d46565.18c27276044cfdbd namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: 7c72e2c5-871a-4d66-b886-3cd515d0b900 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d1ffd-predictor namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: 98d36b91-e0d7-452d-8d14-7025b3defaaf kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: Scaled up replica set error-404-isvc-d1ffd-predictor-bb8d46565 from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:33Z" name: error-404-isvc-d1ffd-predictor.18c2727602116188 namespace: kserve-ci-e2e-test resourceVersion: "26146" uid: 847afa12-6b6a-4ce1-9b85-9a44a6fc86a3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:17:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1ffd-predictor namespace: kserve-ci-e2e-test resourceVersion: "26143" uid: c4bce63e-9344-4013-b4f6-64fb31a7326e kind: Event lastTimestamp: "2026-07-15T11:18:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:03Z" name: error-404-isvc-d1ffd-predictor.18c27279808067ae namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 1a27846e-5bc8-452e-ae56-68921041d402 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:17:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1ffd-predictor namespace: kserve-ci-e2e-test resourceVersion: "26143" uid: c4bce63e-9344-4013-b4f6-64fb31a7326e kind: Event lastTimestamp: "2026-07-15T11:18:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:03Z" name: error-404-isvc-d1ffd-predictor.18c2727980dd0fd5 namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: 16c16d01-a8e5-4395-8c0a-51c2eb91962c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:18:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1ffd-predictor namespace: kserve-ci-e2e-test resourceVersion: "26329" uid: c4bce63e-9344-4013-b4f6-64fb31a7326e kind: Event lastTimestamp: "2026-07-15T11:18:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:48Z" name: error-404-isvc-d1ffd-predictor.18c272807e4c8f53 namespace: kserve-ci-e2e-test resourceVersion: "26907" uid: 1ff99ca0-5a87-4a83-bd6c-c776b7405c45 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:18:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1ffd-predictor namespace: kserve-ci-e2e-test resourceVersion: "26329" uid: c4bce63e-9344-4013-b4f6-64fb31a7326e kind: Event lastTimestamp: "2026-07-15T11:18:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:48Z" name: error-404-isvc-d1ffd-predictor.18c272807eb744b0 namespace: kserve-ci-e2e-test resourceVersion: "26908" uid: df687e29-a5b0-424e-92e6-4c685fc12c10 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1ffd namespace: kserve-ci-e2e-test resourceVersion: "26125" uid: 478fe6bd-271e-476d-a489-89e1119a57fd kind: Event lastTimestamp: "2026-07-15T11:18:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:18:21Z" name: error-404-isvc-d1ffd.18c272760256237b namespace: kserve-ci-e2e-test resourceVersion: "26671" uid: 2dba1075-b7ff-4970-afb0-b56661b4ed30 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1ffd namespace: kserve-ci-e2e-test resourceVersion: "26125" uid: 478fe6bd-271e-476d-a489-89e1119a57fd kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: 'Failed to update status for InferenceService "error-404-isvc-d1ffd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d1ffd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:33Z" name: error-404-isvc-d1ffd.18c2727602b27775 namespace: kserve-ci-e2e-test resourceVersion: "26150" uid: f2d35475-7a6d-4078-aaba-faf147ab34a6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1ffd namespace: kserve-ci-e2e-test resourceVersion: "26125" uid: 478fe6bd-271e-476d-a489-89e1119a57fd kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d1ffd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:33Z" name: error-404-isvc-d1ffd.18c2727602b2c87f namespace: kserve-ci-e2e-test resourceVersion: "26152" uid: 65b7b095-8ede-44da-bf70-0818660f5a17 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1ffd namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: 478fe6bd-271e-476d-a489-89e1119a57fd kind: Event lastTimestamp: "2026-07-15T11:18:21Z" message: InferenceService [error-404-isvc-d1ffd] is Ready metadata: creationTimestamp: "2026-07-15T11:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:18:21Z" name: error-404-isvc-d1ffd.18c2728129ec220b namespace: kserve-ci-e2e-test resourceVersion: "26667" uid: 34c284f5-6b47-4f35-968c-969ae6b33f57 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:07:08.503988Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d6666-predictor-774b85585f-t7hr4 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: 92cf56c4-3843-4b76-9352-acc8bf60b6a8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d6666-predictor-774b85585f-t7hr4 to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:07:08Z" name: error-404-isvc-d6666-predictor-774b85585f-t7hr4.18c271e479e9f342 namespace: kserve-ci-e2e-test resourceVersion: "20596" uid: 278dea55-1ac0-47b7-aec5-f8e20fa3be4e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d6666-predictor-774b85585f-t7hr4 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 92cf56c4-3843-4b76-9352-acc8bf60b6a8 kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d6666-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:08Z" name: error-404-isvc-d6666-predictor-774b85585f-t7hr4.18c271e48344aa08 namespace: kserve-ci-e2e-test resourceVersion: "20608" uid: 08c184c1-3303-420e-ab51-3fcc647f00a2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d6666-predictor-774b85585f-t7hr4 namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: 92cf56c4-3843-4b76-9352-acc8bf60b6a8 kind: Event lastTimestamp: "2026-07-15T11:07:09Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:07:09Z" name: error-404-isvc-d6666-predictor-774b85585f-t7hr4.18c271e4b7166847 namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: e82ee0b6-ac50-47f0-8d9c-25e611743782 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d6666-predictor-774b85585f-t7hr4 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 92cf56c4-3843-4b76-9352-acc8bf60b6a8 kind: Event lastTimestamp: "2026-07-15T11:07:09Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:09Z" name: error-404-isvc-d6666-predictor-774b85585f-t7hr4.18c271e4b8f32a26 namespace: kserve-ci-e2e-test resourceVersion: "20630" uid: 4638d116-ef57-4f9f-8791-8046d216d15f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d6666-predictor-774b85585f-t7hr4 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 92cf56c4-3843-4b76-9352-acc8bf60b6a8 kind: Event lastTimestamp: "2026-07-15T11:07:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:09Z" name: error-404-isvc-d6666-predictor-774b85585f-t7hr4.18c271e4bbc5daba namespace: kserve-ci-e2e-test resourceVersion: "20631" uid: bf044bf6-b8f7-4942-a18f-07add6082fb0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d6666-predictor-774b85585f-t7hr4 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 92cf56c4-3843-4b76-9352-acc8bf60b6a8 kind: Event lastTimestamp: "2026-07-15T11:07:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:09Z" name: error-404-isvc-d6666-predictor-774b85585f-t7hr4.18c271e4bc38d092 namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: cb9797f6-39bf-4c79-ae81-99ba8e50bd6f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d6666-predictor-774b85585f-t7hr4 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 92cf56c4-3843-4b76-9352-acc8bf60b6a8 kind: Event lastTimestamp: "2026-07-15T11:07:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:09Z" name: error-404-isvc-d6666-predictor-774b85585f-t7hr4.18c271e4bc41c5d2 namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 29113de8-7b54-46e1-9116-cf4687333cb9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d6666-predictor-774b85585f-t7hr4 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 92cf56c4-3843-4b76-9352-acc8bf60b6a8 kind: Event lastTimestamp: "2026-07-15T11:07:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:09Z" name: error-404-isvc-d6666-predictor-774b85585f-t7hr4.18c271e4c0a22536 namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: f98b1095-950f-4c6b-9157-f280cfff2908 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d6666-predictor-774b85585f-t7hr4 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 92cf56c4-3843-4b76-9352-acc8bf60b6a8 kind: Event lastTimestamp: "2026-07-15T11:07:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:09Z" name: error-404-isvc-d6666-predictor-774b85585f-t7hr4.18c271e4c11419df namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: c9ccac9d-cf01-4b01-a8f3-a6a6668ebffc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-15T11:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d6666-predictor-774b85585f-t7hr4 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 92cf56c4-3843-4b76-9352-acc8bf60b6a8 kind: Event lastTimestamp: "2026-07-15T11:16:25Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:25Z" name: error-404-isvc-d6666-predictor-774b85585f-t7hr4.18c271e4c94f1b5e namespace: kserve-ci-e2e-test resourceVersion: "25015" uid: 019b5488-908e-4a1c-8ce6-185b6f352f99 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d6666-predictor-774b85585f-t7hr4 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 92cf56c4-3843-4b76-9352-acc8bf60b6a8 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-d6666-predictor-774b85585f-t7hr4.18c272659e55bb55 namespace: kserve-ci-e2e-test resourceVersion: "24953" uid: f161059f-dee7-4352-af0d-982a0137e70b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d6666-predictor-774b85585f-t7hr4 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 92cf56c4-3843-4b76-9352-acc8bf60b6a8 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: error-404-isvc-d6666-predictor-774b85585f-t7hr4.18c272659e566653 namespace: kserve-ci-e2e-test resourceVersion: "24956" uid: ee6093dc-c0da-4112-bf49-f4db045d79ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d6666-predictor-774b85585f-t7hr4 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 92cf56c4-3843-4b76-9352-acc8bf60b6a8 kind: Event lastTimestamp: "2026-07-15T11:16:25Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:25Z" name: error-404-isvc-d6666-predictor-774b85585f-t7hr4.18c272663dd09fcf namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 30d4b822-9786-4f9b-a102-f222df979535 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d6666-predictor-774b85585f namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: 64ba1dad-8e5a-406b-ae12-08eb4690ecd5 kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: 'Created pod: error-404-isvc-d6666-predictor-774b85585f-t7hr4' metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:08Z" name: error-404-isvc-d6666-predictor-774b85585f.18c271e4796dbd74 namespace: kserve-ci-e2e-test resourceVersion: "20593" uid: 8c20a818-4f20-491e-9310-089a59a5f21f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d6666-predictor namespace: kserve-ci-e2e-test resourceVersion: "20583" uid: 16a706ab-159d-44a1-bbf3-b9c1a189794e kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: Scaled up replica set error-404-isvc-d6666-predictor-774b85585f from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:08Z" name: error-404-isvc-d6666-predictor.18c271e4771a4ed2 namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: 2703c9ff-a9bf-480e-853b-0f090c91eeb7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-15T11:07:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d6666-predictor namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: ef382360-2585-41eb-898b-b641877ce04f kind: Event lastTimestamp: "2026-07-15T11:08:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:07:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:23Z" name: error-404-isvc-d6666-predictor.18c271e7f5808e00 namespace: kserve-ci-e2e-test resourceVersion: "21507" uid: b92e2d27-cdc0-437a-b12a-be1966df5e29 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-15T11:07:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d6666-predictor namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: ef382360-2585-41eb-898b-b641877ce04f kind: Event lastTimestamp: "2026-07-15T11:08:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:07:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:23Z" name: error-404-isvc-d6666-predictor.18c271e7f5f138c3 namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: cafa65ca-2d9f-437d-bce1-999c0e34de77 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d6666-predictor namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: ef382360-2585-41eb-898b-b641877ce04f kind: Event lastTimestamp: "2026-07-15T11:07:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:38Z" name: error-404-isvc-d6666-predictor.18c271eb755c1c88 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: f4488d30-f0b3-49a9-a855-de59e257faea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d6666-predictor namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: ef382360-2585-41eb-898b-b641877ce04f kind: Event lastTimestamp: "2026-07-15T11:07:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:38Z" name: error-404-isvc-d6666-predictor.18c271eb75d5c3ac namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: 4526c1f7-785f-4663-a038-a5a3a6c4fa8e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d6666 namespace: kserve-ci-e2e-test resourceVersion: "20573" uid: 126fb114-6588-4702-95d0-234a9abd08db kind: Event lastTimestamp: "2026-07-15T11:07:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:07:55Z" name: error-404-isvc-d6666.18c271e4771ca40b namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: c246e513-a6e2-4156-a663-fc50daa8634c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d6666 namespace: kserve-ci-e2e-test resourceVersion: "20573" uid: 126fb114-6588-4702-95d0-234a9abd08db kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: 'Failed to update status for InferenceService "error-404-isvc-d6666": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d6666": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:07:08Z" name: error-404-isvc-d6666.18c271e47813a926 namespace: kserve-ci-e2e-test resourceVersion: "20589" uid: 2b7f83fc-29c4-4d55-a836-c59e516754ff reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d6666 namespace: kserve-ci-e2e-test resourceVersion: "20573" uid: 126fb114-6588-4702-95d0-234a9abd08db kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d6666": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:07:08Z" name: error-404-isvc-d6666.18c271e47813ea3e namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: fb353a0b-7303-47d3-a6aa-091cf365e1dc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d6666 namespace: kserve-ci-e2e-test resourceVersion: "21226" uid: 126fb114-6588-4702-95d0-234a9abd08db kind: Event lastTimestamp: "2026-07-15T11:07:55Z" message: InferenceService [error-404-isvc-d6666] is Ready metadata: creationTimestamp: "2026-07-15T11:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:07:55Z" name: error-404-isvc-d6666.18c271ef84f5b41a namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: 55025d9e-76e5-4e54-99cf-454dda04a565 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T10:56:33.296799Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: ae6f7d2a-dde4-4d1a-8931-9f1434e352f7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg.18c27150949d678c namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: b0a281ea-4a6b-4f4c-a31b-55243699674d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg namespace: kserve-ci-e2e-test resourceVersion: "15242" uid: ae6f7d2a-dde4-4d1a-8931-9f1434e352f7 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg.18c27150aeb88eaa namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 9c2c149b-5a22-4604-b073-2c13fb3b7a46 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: ae6f7d2a-dde4-4d1a-8931-9f1434e352f7 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg.18c27150b063429e namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: 2b54f9a3-bef7-42a2-bd0a-97402d44bba6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: ae6f7d2a-dde4-4d1a-8931-9f1434e352f7 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg.18c27150b3000c82 namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: 1c12522a-a173-49ae-b5c8-aea97b13327b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: ae6f7d2a-dde4-4d1a-8931-9f1434e352f7 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg.18c27150b3608eb0 namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 945c677d-41bc-4a0b-afee-52bbd5302a13 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: ae6f7d2a-dde4-4d1a-8931-9f1434e352f7 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg.18c27150b372473d namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: f4da89d7-6606-4384-ad84-684e38057854 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: ae6f7d2a-dde4-4d1a-8931-9f1434e352f7 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg.18c27150b635689d namespace: kserve-ci-e2e-test resourceVersion: "15274" uid: 7da083f2-86ae-436a-9091-bead16f7dc17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: ae6f7d2a-dde4-4d1a-8931-9f1434e352f7 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg.18c27150b69a6de6 namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: 78265377-e697-4958-99d3-e72b2f771b21 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-15T10:56:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: ae6f7d2a-dde4-4d1a-8931-9f1434e352f7 kind: Event lastTimestamp: "2026-07-15T11:05:50Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:50Z" name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg.18c27150cf73e5aa namespace: kserve-ci-e2e-test resourceVersion: "19676" uid: a9270ff9-695a-415e-9f34-0956109e6054 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: ae6f7d2a-dde4-4d1a-8931-9f1434e352f7 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg.18c271d1bc4c4746 namespace: kserve-ci-e2e-test resourceVersion: "19610" uid: a2bfd92c-48da-42aa-931c-af760dfdd539 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: ae6f7d2a-dde4-4d1a-8931-9f1434e352f7 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg.18c271d1bc4d9b7a namespace: kserve-ci-e2e-test resourceVersion: "19615" uid: 8850a936-3994-4337-bd0e-0132db9a02b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: ae6f7d2a-dde4-4d1a-8931-9f1434e352f7 kind: Event lastTimestamp: "2026-07-15T11:05:50Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:50Z" name: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg.18c271d24416410f namespace: kserve-ci-e2e-test resourceVersion: "19675" uid: 64c60c02-2fe3-49a7-a03a-e2654ebb2dcb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ff4f8-predictor-557dbbf4ff namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 5084f0b7-8896-4736-88cc-da3ce08e1ab0 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'Created pod: error-404-isvc-ff4f8-predictor-557dbbf4ff-fb8lg' metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-ff4f8-predictor-557dbbf4ff.18c27150943d5226 namespace: kserve-ci-e2e-test resourceVersion: "15230" uid: 2a268405-3fa4-42cc-9526-85a86866807e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ff4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15206" uid: d5fc4aca-8784-4965-904b-2e02ee1ae9c0 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Scaled up replica set error-404-isvc-ff4f8-predictor-557dbbf4ff from 0 to 1 metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-ff4f8-predictor.18c2715092957e8d namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: 3232029f-559e-4b82-a8be-2e9da263b1fa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T10:56:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ff4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15211" uid: 21980276-c6fb-4d27-90db-5e5f0a478d05 kind: Event lastTimestamp: "2026-07-15T10:57:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:03Z" name: error-404-isvc-ff4f8-predictor.18c2715411a71dd0 namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: d5e6aaff-b0de-4ab3-a07a-cc6d4b36b013 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T10:56:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ff4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15211" uid: 21980276-c6fb-4d27-90db-5e5f0a478d05 kind: Event lastTimestamp: "2026-07-15T10:57:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:03Z" name: error-404-isvc-ff4f8-predictor.18c2715412a3d60a namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: bed0ac5b-a0c1-406c-af30-28948e4a7fb7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:57:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ff4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 21980276-c6fb-4d27-90db-5e5f0a478d05 kind: Event lastTimestamp: "2026-07-15T10:57:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:48Z" name: error-404-isvc-ff4f8-predictor.18c2715b103571e9 namespace: kserve-ci-e2e-test resourceVersion: "16168" uid: 4ee7d817-f61a-4a17-ac4e-ee6a4ae70b09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:57:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ff4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 21980276-c6fb-4d27-90db-5e5f0a478d05 kind: Event lastTimestamp: "2026-07-15T10:57:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:48Z" name: error-404-isvc-ff4f8-predictor.18c2715b10a34cce namespace: kserve-ci-e2e-test resourceVersion: "16169" uid: b89bf335-58b0-47ac-b360-d12162ab62e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ff4f8 namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 15a4f88f-49a3-47d3-a5a4-12beea1c4d01 kind: Event lastTimestamp: "2026-07-15T10:57:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:57:20Z" name: error-404-isvc-ff4f8.18c27150929f8755 namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: cd8240af-e8c4-4ba5-bf0e-bd6181216db3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ff4f8 namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 15a4f88f-49a3-47d3-a5a4-12beea1c4d01 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'Failed to update status for InferenceService "error-404-isvc-ff4f8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ff4f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-ff4f8.18c27150932a0aca namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: adfc8333-b7e1-4f0a-986d-488c1df6fd7c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ff4f8 namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 15a4f88f-49a3-47d3-a5a4-12beea1c4d01 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ff4f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:56:33Z" name: error-404-isvc-ff4f8.18c27150932a7850 namespace: kserve-ci-e2e-test resourceVersion: "15225" uid: c2795ca3-41f1-459a-8ae9-e6c7962cb58b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ff4f8 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: 15a4f88f-49a3-47d3-a5a4-12beea1c4d01 kind: Event lastTimestamp: "2026-07-15T10:57:20Z" message: InferenceService [error-404-isvc-ff4f8] is Ready metadata: creationTimestamp: "2026-07-15T10:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:57:20Z" name: error-404-isvc-ff4f8.18c2715b8af83d1e namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: effe8551-289d-47f1-a133-61ae0d42b0f5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T10:54:43.180115Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13997" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T10:54:43Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c27136f1263791 namespace: kserve-ci-e2e-test resourceVersion: "14001" uid: 532bc057-3dbb-44ac-97dc-14cc194f2a7a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "14008" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T10:54:43Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c271370b0699ff namespace: kserve-ci-e2e-test resourceVersion: "14066" uid: 197a3b42-7efd-439b-af32-eacaabbb45a9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:523ed64da171f96ba586fa05d2bcb415c8c54dddf273afd4079bdc377a22c2e7" already present on machine metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:43Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c271370cba6e21 namespace: kserve-ci-e2e-test resourceVersion: "14075" uid: 1971d494-8a1c-4bcc-9afa-a051d5dd8e64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:43Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c271370f98ecf7 namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: d8a6ea5e-17b1-4ea3-824a-53534ef194e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:43Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c271370ffcf636 namespace: kserve-ci-e2e-test resourceVersion: "14084" uid: ce313f1b-fb69-4e39-a692-d74f7b3c3549 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:54:50Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1738" metadata: creationTimestamp: "2026-07-15T10:54:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:50Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c27138b886bebc namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 2dc851b3-6f16-4be8-aeba-1814c38d8d27 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1738" in 11.454s (11.454s including waiting). Image size: 1555868070 bytes.' metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c2713b633ec4c9 namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: 4f783a25-09a4-4d2f-a34a-ccf9c34b6605 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c2713b6745f94b namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: 006f57fa-a2e7-4c51-b69c-212db89c5900 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c2713b6848eaf2 namespace: kserve-ci-e2e-test resourceVersion: "14304" uid: 24c244b7-3481-4e00-b658-50e321167faf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c2713b686297b3 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: 27926823-e060-47f1-82f8-72f526b780d3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:55:07Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 5.111s (5.111s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:07Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c2713c991064d6 namespace: kserve-ci-e2e-test resourceVersion: "14344" uid: c6d32ee8-5a73-45a6-ae7a-fb25ab369159 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:55:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:07Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c2713c9ea70891 namespace: kserve-ci-e2e-test resourceVersion: "14347" uid: 7dfb1f74-9b80-479c-b81f-c106a1798e07 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:55:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:07Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c2713c9fd6cfe5 namespace: kserve-ci-e2e-test resourceVersion: "14349" uid: 43a2b4e7-7396-45b6-adfd-1359e2a0e8fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-15T10:55:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:57:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:14Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c2713cf03e75c3 namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: dc449d20-3bfb-4e20-9e47-8279f884661b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c27159d6ba730e namespace: kserve-ci-e2e-test resourceVersion: "15720" uid: 0a5bb7bc-8281-4d06-883c-e81d01c5cc21 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c27159d6bafd69 namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: 0a28e93c-98cf-4992-8329-74d126153e39 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 0d952e58-6e36-4634-8acd-fdc7dd140fc3 kind: Event lastTimestamp: "2026-07-15T10:57:14Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:14Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb.18c2715a46b3916e namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: 74b086af-82f5-4986-bf2d-a5a52fa4a70a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-5cb7bb944 namespace: kserve-ci-e2e-test resourceVersion: "13986" uid: e89045f1-e275-4005-af3d-d28a967d36de kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-5cb7bb944-h7qvb' metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:54:43Z" name: isvc-sklearn-graph-1-predictor-5cb7bb944.18c27136f00cc117 namespace: kserve-ci-e2e-test resourceVersion: "14000" uid: 3e7b8525-78ea-4459-b87e-a5ef9d925eac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13982" uid: 80ebbad3-4be6-4532-ac1a-686137c67f08 kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-5cb7bb944 from 0 to 1 metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:54:43Z" name: isvc-sklearn-graph-1-predictor.18c27136e975005e namespace: kserve-ci-e2e-test resourceVersion: "13990" uid: 0e1e1da8-eac4-418a-8cdf-af05a7eb63f2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:54:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: 3ca9f5d7-dfab-4aeb-b97a-7969c2dadb4e kind: Event lastTimestamp: "2026-07-15T10:55:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:55:28Z" name: isvc-sklearn-graph-1-predictor.18c2713a67eeb939 namespace: kserve-ci-e2e-test resourceVersion: "14545" uid: d7e4b106-708c-41c9-9ce5-cbf2d26bfd3f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:54:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: 3ca9f5d7-dfab-4aeb-b97a-7969c2dadb4e kind: Event lastTimestamp: "2026-07-15T10:55:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:55:28Z" name: isvc-sklearn-graph-1-predictor.18c2713a687f363a namespace: kserve-ci-e2e-test resourceVersion: "14546" uid: ba08e38d-d7f9-4400-8caa-4de130560103 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T10:55:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 3ca9f5d7-dfab-4aeb-b97a-7969c2dadb4e kind: Event lastTimestamp: "2026-07-15T10:56:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:58Z" name: isvc-sklearn-graph-1-predictor.18c27144e52338b9 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: d9788389-7eb7-407a-81cd-bc8bf81d417b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T10:55:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 3ca9f5d7-dfab-4aeb-b97a-7969c2dadb4e kind: Event lastTimestamp: "2026-07-15T10:56:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:58Z" name: isvc-sklearn-graph-1-predictor.18c27144e598ead3 namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: 48f959e9-087f-4c19-9de1-3416dc57875a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13971" uid: 79c85774-5513-412f-858d-902b488e6905 kind: Event lastTimestamp: "2026-07-15T10:56:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:56:35Z" name: isvc-sklearn-graph-1.18c27136e949124e namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 5856bc36-1d44-46cd-a24c-6300629c37e3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15300" uid: 79c85774-5513-412f-858d-902b488e6905 kind: Event lastTimestamp: "2026-07-15T10:56:35Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-15T10:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:56:35Z" name: isvc-sklearn-graph-1.18c27150fcd00cf6 namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: efeb8992-092d-4f2f-9b46-d4c6615c4923 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T10:54:43.704359Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T10:54:43Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2713710659447 namespace: kserve-ci-e2e-test resourceVersion: "14085" uid: 90665074-8ce0-46d6-ba88-50704b01cf8d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:43Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2713720aba7a7 namespace: kserve-ci-e2e-test resourceVersion: "14105" uid: aaabc854-7619-44fc-9dce-e423e2e88d7a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14092" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:54:44Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T10:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T10:54:44Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2713751420f96 namespace: kserve-ci-e2e-test resourceVersion: "14132" uid: 07badd26-f767-442d-8a3b-4afeb143c4b7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:54:44Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:523ed64da171f96ba586fa05d2bcb415c8c54dddf273afd4079bdc377a22c2e7" already present on machine metadata: creationTimestamp: "2026-07-15T10:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:44Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c27137533ce7a1 namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 3ead15ca-d0a4-49e2-8ac7-ff4c2e8b6947 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:54:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-15T10:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:44Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2713756ae0f74 namespace: kserve-ci-e2e-test resourceVersion: "14137" uid: 7597b6bf-8e69-404d-8be7-90cb342d0951 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:54:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-15T10:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:44Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2713757a93c5f namespace: kserve-ci-e2e-test resourceVersion: "14139" uid: fae75d5c-4e28-4da6-93d5-0d4241103f64 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:54:50Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1738" metadata: creationTimestamp: "2026-07-15T10:54:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:50Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c27138b8ac953d namespace: kserve-ci-e2e-test resourceVersion: "14183" uid: ac7f21a4-fa4b-4f3f-9b53-0a75ae1f8ecd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1738" in 11.441s (11.441s including waiting). Image size: 1555868070 bytes.' metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2713b629e0a8c namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 39312183-ee77-426b-8f0d-69bab52e009f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2713b678673ee namespace: kserve-ci-e2e-test resourceVersion: "14303" uid: 4299cbb1-f88b-4c81-b28d-2c1b84cbc235 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2713b6891fb14 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: f32bbdda-1b20-487f-8593-a768a15e2975 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2713b68a2c826 namespace: kserve-ci-e2e-test resourceVersion: "14307" uid: 2a5921b5-931f-4482-a0df-5ae9c59920b1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:55:07Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 5.4s (5.4s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:07Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2713caa8bf1b6 namespace: kserve-ci-e2e-test resourceVersion: "14351" uid: 50dce3e2-634f-4ff6-a26f-4a165129f10d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:55:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:07Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2713caeebc356 namespace: kserve-ci-e2e-test resourceVersion: "14354" uid: 3b2393c3-1a28-44e9-859e-7e403b8ddf8a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:55:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:07Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2713cafe69b87 namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: 07627534-d0a3-4f42-9ba8-5c22cc0b54a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2713cb4229fcd namespace: kserve-ci-e2e-test resourceVersion: "15803" uid: 28e55b0f-6499-45ab-b1f6-a328520ae77f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c27159d35ef41e namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: 5e38cc61-7a28-4405-b974-61dc705c7935 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c27159d35f76a9 namespace: kserve-ci-e2e-test resourceVersion: "15693" uid: f9456ba9-e3ec-4010-9f95-4ba7187cb44d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh namespace: kserve-ci-e2e-test resourceVersion: "14082" uid: da453fef-0f6a-43b1-a04a-e5fabd061460 kind: Event lastTimestamp: "2026-07-15T10:57:18Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:18Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh.18c2715a0accc680 namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: 8e023839-23c5-4ff9-bb9a-241478f52d07 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-d8594dbb4 namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: 271ae0d9-abf6-4f32-8929-93eb31da1e35 kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-d8594dbb4-w5wlh' metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:54:43Z" name: isvc-sklearn-graph-2-predictor-d8594dbb4.18c271370fc76155 namespace: kserve-ci-e2e-test resourceVersion: "14081" uid: 93b487b5-ffa8-48b8-8e8b-e41622039e2e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14065" uid: 690ff08b-e2e8-4c97-8393-324f95edfd33 kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-d8594dbb4 from 0 to 1 metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:54:43Z" name: isvc-sklearn-graph-2-predictor.18c271370bcf8da1 namespace: kserve-ci-e2e-test resourceVersion: "14070" uid: 00dbb9a2-c232-4ed8-84f3-d9f4f15724f6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:54:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14067" uid: 594732f2-adfc-4370-85c0-71c55f433489 kind: Event lastTimestamp: "2026-07-15T10:55:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:55:28Z" name: isvc-sklearn-graph-2-predictor.18c2713a8aebceb4 namespace: kserve-ci-e2e-test resourceVersion: "14556" uid: 24ee297c-aabf-4953-b95c-138809a5cdd3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:54:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14067" uid: 594732f2-adfc-4370-85c0-71c55f433489 kind: Event lastTimestamp: "2026-07-15T10:55:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:55:28Z" name: isvc-sklearn-graph-2-predictor.18c2713a8bc4323d namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: 20c8fcb5-08ac-490e-ac14-566924415f99 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T10:55:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: 594732f2-adfc-4370-85c0-71c55f433489 kind: Event lastTimestamp: "2026-07-15T10:56:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:58Z" name: isvc-sklearn-graph-2-predictor.18c271450827ab97 namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: a61f00c8-a770-47eb-bfa9-105098b82741 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T10:55:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: 594732f2-adfc-4370-85c0-71c55f433489 kind: Event lastTimestamp: "2026-07-15T10:56:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:58Z" name: isvc-sklearn-graph-2-predictor.18c271450899a4d0 namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: 31acbbe9-f617-472a-9f1d-1bbb53c9e1a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14049" uid: d3d8e9d9-b067-4101-8fce-5666834b9403 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:56:33Z" name: isvc-sklearn-graph-2.18c271370bd177cc namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: f7964068-a27a-4b70-bb34-b161e2be32a5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15263" uid: d3d8e9d9-b067-4101-8fce-5666834b9403 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:56:33Z" name: isvc-sklearn-graph-2.18c27150b0a2b7bf namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: 16d2caf0-c645-4853-a297-9e7b9b07e2af reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T10:54:43.560055Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-j57gn to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T10:54:43Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c2713707cba2b0 namespace: kserve-ci-e2e-test resourceVersion: "14054" uid: a7d3902b-de69-4578-a0f8-f53f24aaa4c4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14061" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:54:44Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T10:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T10:54:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c2713728cf2d70 namespace: kserve-ci-e2e-test resourceVersion: "14113" uid: b8b30887-5ab6-467e-b399-fe4cec6dec28 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:54:44Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:523ed64da171f96ba586fa05d2bcb415c8c54dddf273afd4079bdc377a22c2e7" already present on machine metadata: creationTimestamp: "2026-07-15T10:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c271372b5f8636 namespace: kserve-ci-e2e-test resourceVersion: "14116" uid: 5b8432d0-7e58-4871-9305-52f9c7af8597 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:54:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-15T10:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c271372fd653d7 namespace: kserve-ci-e2e-test resourceVersion: "14120" uid: ac8e24eb-804b-4a40-afee-e671f896f609 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:54:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-15T10:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c27137317fed29 namespace: kserve-ci-e2e-test resourceVersion: "14121" uid: 49879a4b-68fd-409e-a7a6-53bffdfe36c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:54:50Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-15T10:54:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:50Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c27138b86a54ae namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: ab6a1d21-b110-493d-851c-c637ecf45bcf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:55:22Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 31.346s (31.346s including waiting). Image size: 1306898171 bytes.' metadata: creationTimestamp: "2026-07-15T10:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c2714004cc793f namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: 87ac3878-61d4-4ca0-a20a-62f26b27b036 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:55:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T10:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c2714007eb2c62 namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: f0b9fe1e-10d7-40f0-af31-7e292369cfd0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:55:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T10:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c2714008642657 namespace: kserve-ci-e2e-test resourceVersion: "14490" uid: 0d07121c-148b-4c12-87e8-949ba3f56f78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:55:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T10:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c27140088162f3 namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 295e97e8-923a-4f0f-8dd6-a4361f4b2c67 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:55:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T10:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c271400b950c6d namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: ba8ecc87-65a5-4803-8999-e2a9d4b82aef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:55:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c271400c0aef63 namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: bfd9a03a-691c-483d-a5c4-d7ff551f57d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-15T10:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:56:20Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c27140727991cc namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: b19bdca0-8362-4bb5-b119-52d2ec2fdd57 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c27159db428b90 namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: 01e2bb49-e4fc-4e4f-8cda-272d725d5f61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c27159db42f07a namespace: kserve-ci-e2e-test resourceVersion: "15755" uid: 2bbf448f-0c09-4f8e-8ec6-cf26bccac178 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-j57gn namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 478795bd-bfa8-449e-8ba0-e9d93d900364 kind: Event lastTimestamp: "2026-07-15T10:57:15Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-j57gn.18c2715a4acf9bbc namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: f8a8e75d-d0c6-4927-8bb5-acfb7f7ff48a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14041" uid: dcd4f260-0b06-4555-9df0-b80487f2671c kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-j57gn' metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:54:43Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c27137073f6c3f namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: a3c67567-3e3e-4bd6-8de8-47d5cde1038e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14040" uid: 25d9b3af-2d55-41ef-8ad0-1bd9205d6b64 kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:54:43Z" name: isvc-xgboost-graph-predictor.18c2713700a391b0 namespace: kserve-ci-e2e-test resourceVersion: "14043" uid: 3ff2bbae-68c8-420b-b434-44219c994038 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-15T10:54:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14042" uid: ad95eb22-380b-43d3-911a-ed252a710518 kind: Event lastTimestamp: "2026-07-15T10:55:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:55:58Z" name: isvc-xgboost-graph-predictor.18c2713a81c66929 namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: b6095f3a-29d1-4f6e-9905-a3182d857df9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-15T10:54:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14042" uid: ad95eb22-380b-43d3-911a-ed252a710518 kind: Event lastTimestamp: "2026-07-15T10:55:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:55:58Z" name: isvc-xgboost-graph-predictor.18c2713a826293c8 namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: e1525bf1-f46d-4414-8cef-e3ac43a83c56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T10:55:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14262" uid: ad95eb22-380b-43d3-911a-ed252a710518 kind: Event lastTimestamp: "2026-07-15T10:56:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:58Z" name: isvc-xgboost-graph-predictor.18c27144fec14873 namespace: kserve-ci-e2e-test resourceVersion: "15497" uid: d8bbd3c9-4bda-44cf-ad74-a1d533e344bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T10:55:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14262" uid: ad95eb22-380b-43d3-911a-ed252a710518 kind: Event lastTimestamp: "2026-07-15T10:56:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:58Z" name: isvc-xgboost-graph-predictor.18c27144ff34f7bc namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: 2749b984-e134-4c55-a603-732554e5be41 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14025" uid: 2db22d3e-6277-4f3e-b65e-96d9119b9870 kind: Event lastTimestamp: "2026-07-15T10:56:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:56:30Z" name: isvc-xgboost-graph.18c2713701419d4d namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 04db01bf-11d5-42fc-b092-1a075cbe39ac reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 2db22d3e-6277-4f3e-b65e-96d9119b9870 kind: Event lastTimestamp: "2026-07-15T10:56:30Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-15T10:56:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:56:30Z" name: isvc-xgboost-graph.18c2714fd6dd67e0 namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: 70e13740-8330-4e0f-967d-fc6bc5d2e649 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T10:57:02.809946Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7fdbd8cf64-8vlgf namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: 2b09147f-02ce-48c0-a197-d696e1ab1946 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-7fdbd8cf64-8vlgf to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-07-15T10:57:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T10:57:02Z" name: model-chainer-7fdbd8cf64-8vlgf.18c2715773bc52c7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: eb9e1171-ac5a-4d14-ad08-0bd77046fef8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:03Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7fdbd8cf64-8vlgf namespace: kserve-ci-e2e-test resourceVersion: "15552" uid: 2b09147f-02ce-48c0-a197-d696e1ab1946 kind: Event lastTimestamp: "2026-07-15T10:57:03Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T10:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T10:57:03Z" name: model-chainer-7fdbd8cf64-8vlgf.18c271578dd22473 namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: fb72e678-5443-46b8-970b-a2cd5481294e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7fdbd8cf64-8vlgf namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 2b09147f-02ce-48c0-a197-d696e1ab1946 kind: Event lastTimestamp: "2026-07-15T10:57:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0e1a169e211c4cdcd19b5d9ae2b04ed590c1bcb2f57424f5c6e7d0d0f2faa1f0" already present on machine metadata: creationTimestamp: "2026-07-15T10:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:03Z" name: model-chainer-7fdbd8cf64-8vlgf.18c271578f1cdd17 namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: 1dcb4382-caf2-43dd-95f2-eabd46055c7e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7fdbd8cf64-8vlgf namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 2b09147f-02ce-48c0-a197-d696e1ab1946 kind: Event lastTimestamp: "2026-07-15T10:57:03Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-15T10:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:03Z" name: model-chainer-7fdbd8cf64-8vlgf.18c2715792ea5227 namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 95a22f10-e536-4d1b-9351-df9d21eea8cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7fdbd8cf64-8vlgf namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 2b09147f-02ce-48c0-a197-d696e1ab1946 kind: Event lastTimestamp: "2026-07-15T10:57:03Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-15T10:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:03Z" name: model-chainer-7fdbd8cf64-8vlgf.18c271579362c429 namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 57a94ee4-a6ed-4f0d-9a27-2971eaca06d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7fdbd8cf64-8vlgf namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 2b09147f-02ce-48c0-a197-d696e1ab1946 kind: Event lastTimestamp: "2026-07-15T10:57:12Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-15T10:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:12Z" name: model-chainer-7fdbd8cf64-8vlgf.18c27159cd3116cd namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: 93d144b8-9196-4165-9e75-2639f2561825 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T10:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7fdbd8cf64-8vlgf namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 2b09147f-02ce-48c0-a197-d696e1ab1946 kind: Event lastTimestamp: "2026-07-15T10:57:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-15T10:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:39Z" name: model-chainer-7fdbd8cf64-8vlgf.18c2715a3e97be50 namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: 927bdc10-9236-4df6-9025-7a38d8eefe99 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-7fdbd8cf64 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: e38e5acb-4d59-4499-9d4f-05d3e1b1b3b8 kind: Event lastTimestamp: "2026-07-15T10:57:02Z" message: 'Created pod: model-chainer-7fdbd8cf64-8vlgf' metadata: creationTimestamp: "2026-07-15T10:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:02Z" name: model-chainer-7fdbd8cf64.18c27157734eda41 namespace: kserve-ci-e2e-test resourceVersion: "15544" uid: fb413297-d209-41fa-8120-8fb4eacf2e92 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: a129e1e1-d568-47db-be80-ec132ccbdd34 kind: Event lastTimestamp: "2026-07-15T10:57:02Z" message: Scaled up replica set model-chainer-7fdbd8cf64 from 0 to 1 metadata: creationTimestamp: "2026-07-15T10:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:02Z" name: model-chainer.18c2715772641801 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: b809c7ec-81c2-4cad-8fcb-20b001054287 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15554" uid: f0fb3368-dc06-4f55-b22b-8fe88e86e894 kind: Event lastTimestamp: "2026-07-15T10:57:02Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-15T10:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:57:02Z" name: model-chainer.18c2715777b59b2b namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 4b85f1b5-35ae-42c2-b58e-59860e0fd926 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T10:58:13.139441Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-074d2-574bc49b98-k4s78 namespace: kserve-ci-e2e-test resourceVersion: "16376" uid: 3bfd496a-bb37-4b4a-bad2-c567178570e3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-074d2-574bc49b98-k4s78 to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-07-15T10:58:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T10:58:13Z" name: sequence-graph-074d2-574bc49b98-k4s78.18c27167d3b34c65 namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: c2605b00-c63a-4702-98a5-dadde4013640 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:58:13Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-074d2-574bc49b98-k4s78 namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: 3bfd496a-bb37-4b4a-bad2-c567178570e3 kind: Event lastTimestamp: "2026-07-15T10:58:13Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T10:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T10:58:13Z" name: sequence-graph-074d2-574bc49b98-k4s78.18c27167eda73a31 namespace: kserve-ci-e2e-test resourceVersion: "16400" uid: b1e8cba8-7868-451a-a5f5-53ec8011edd6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-074d2} kind: Pod name: sequence-graph-074d2-574bc49b98-k4s78 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 3bfd496a-bb37-4b4a-bad2-c567178570e3 kind: Event lastTimestamp: "2026-07-15T10:58:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0e1a169e211c4cdcd19b5d9ae2b04ed590c1bcb2f57424f5c6e7d0d0f2faa1f0" already present on machine metadata: creationTimestamp: "2026-07-15T10:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:58:13Z" name: sequence-graph-074d2-574bc49b98-k4s78.18c27167ef248fcb namespace: kserve-ci-e2e-test resourceVersion: "16402" uid: 1e16ac1c-7ab0-44ad-a84c-81367e91c835 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-074d2} kind: Pod name: sequence-graph-074d2-574bc49b98-k4s78 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 3bfd496a-bb37-4b4a-bad2-c567178570e3 kind: Event lastTimestamp: "2026-07-15T10:58:13Z" message: 'Created container: sequence-graph-074d2' metadata: creationTimestamp: "2026-07-15T10:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:58:13Z" name: sequence-graph-074d2-574bc49b98-k4s78.18c27167f288b8ec namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: e0254320-54ed-4854-b148-dfa1294c9095 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-074d2} kind: Pod name: sequence-graph-074d2-574bc49b98-k4s78 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 3bfd496a-bb37-4b4a-bad2-c567178570e3 kind: Event lastTimestamp: "2026-07-15T10:58:13Z" message: Started container sequence-graph-074d2 metadata: creationTimestamp: "2026-07-15T10:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:58:13Z" name: sequence-graph-074d2-574bc49b98-k4s78.18c27167f305fc73 namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 2718e1be-3dee-414a-82ef-a3d84fe6d01a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-074d2} kind: Pod name: sequence-graph-074d2-574bc49b98-k4s78 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 3bfd496a-bb37-4b4a-bad2-c567178570e3 kind: Event lastTimestamp: "2026-07-15T11:06:27Z" message: Stopping container sequence-graph-074d2 metadata: creationTimestamp: "2026-07-15T11:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:27Z" name: sequence-graph-074d2-574bc49b98-k4s78.18c271db03cd42a9 namespace: kserve-ci-e2e-test resourceVersion: "19981" uid: 01975b3c-aa49-48b8-9ad8-3fa4afd582cb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-074d2} kind: Pod name: sequence-graph-074d2-574bc49b98-k4s78 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 3bfd496a-bb37-4b4a-bad2-c567178570e3 kind: Event lastTimestamp: "2026-07-15T11:06:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-15T11:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:55Z" name: sequence-graph-074d2-574bc49b98-k4s78.18c271db8c16ae32 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 3fbcd1eb-48d5-4970-b9bc-a7afdef641af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:58:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-074d2-574bc49b98 namespace: kserve-ci-e2e-test resourceVersion: "16373" uid: 3386fe65-2b51-4ae2-aba1-ab16888c2f57 kind: Event lastTimestamp: "2026-07-15T10:58:13Z" message: 'Created pod: sequence-graph-074d2-574bc49b98-k4s78' metadata: creationTimestamp: "2026-07-15T10:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:58:13Z" name: sequence-graph-074d2-574bc49b98.18c27167d2efc48a namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: 029597c7-1f3c-4939-8347-df16cc75fa91 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:58:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-074d2 namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: d47b5921-b729-414c-9151-6bb41efed648 kind: Event lastTimestamp: "2026-07-15T10:58:13Z" message: Scaled up replica set sequence-graph-074d2-574bc49b98 from 0 to 1 metadata: creationTimestamp: "2026-07-15T10:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:58:13Z" name: sequence-graph-074d2.18c27167d1aa2b8b namespace: kserve-ci-e2e-test resourceVersion: "16374" uid: 14385f4d-6c0c-4bd0-8d12-1aa153c21ab3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:58:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-074d2 namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: 9d316317-b6ac-40e2-8097-9a6ba46d6aff kind: Event lastTimestamp: "2026-07-15T10:58:13Z" message: InferenceGraph [sequence-graph-074d2] is Ready metadata: creationTimestamp: "2026-07-15T10:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:58:13Z" name: sequence-graph-074d2.18c27167d7c947cc namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: 8a3d8d6c-0ffc-4761-94e3-7e74c7d3a26b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T10:58:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-074d2 namespace: kserve-ci-e2e-test resourceVersion: "16372" uid: 75619edc-e92d-4c90-8318-98d4790218f3 kind: Event lastTimestamp: "2026-07-15T10:58:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:58:43Z" name: sequence-graph-074d2.18c2716b50273073 namespace: kserve-ci-e2e-test resourceVersion: "16619" uid: 05e0f141-552b-4b01-964f-307ebf46aabf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T10:58:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-074d2 namespace: kserve-ci-e2e-test resourceVersion: "16372" uid: 75619edc-e92d-4c90-8318-98d4790218f3 kind: Event lastTimestamp: "2026-07-15T10:58:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:58:43Z" name: sequence-graph-074d2.18c2716b50b728ea namespace: kserve-ci-e2e-test resourceVersion: "16620" uid: 46e44fee-7934-4ebc-bea9-531c94b2d8a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:07:38.099395Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-30120-7b65c6b7f-q2rbc namespace: kserve-ci-e2e-test resourceVersion: "20907" uid: c17d742e-9485-43cf-bf97-048f41f53351 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-30120-7b65c6b7f-q2rbc to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-07-15T11:07:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:07:38Z" name: sequence-graph-30120-7b65c6b7f-q2rbc.18c271eb5df0140b namespace: kserve-ci-e2e-test resourceVersion: "20912" uid: bae4bc17-0939-49ec-aca8-5c3eb8ce429a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:38Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-30120-7b65c6b7f-q2rbc namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: c17d742e-9485-43cf-bf97-048f41f53351 kind: Event lastTimestamp: "2026-07-15T11:07:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-30120-serving-cert" not found' metadata: creationTimestamp: "2026-07-15T11:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:38Z" name: sequence-graph-30120-7b65c6b7f-q2rbc.18c271eb69a9d6a1 namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: 9c78d543-9287-439e-a08f-9ad3789bf92f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:39Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-30120-7b65c6b7f-q2rbc namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: c17d742e-9485-43cf-bf97-048f41f53351 kind: Event lastTimestamp: "2026-07-15T11:07:39Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:07:39Z" name: sequence-graph-30120-7b65c6b7f-q2rbc.18c271eb9e985a24 namespace: kserve-ci-e2e-test resourceVersion: "20938" uid: 7aa1ada2-0a9b-44f3-9923-26daa7b44398 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30120} kind: Pod name: sequence-graph-30120-7b65c6b7f-q2rbc namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: c17d742e-9485-43cf-bf97-048f41f53351 kind: Event lastTimestamp: "2026-07-15T11:07:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0e1a169e211c4cdcd19b5d9ae2b04ed590c1bcb2f57424f5c6e7d0d0f2faa1f0" already present on machine metadata: creationTimestamp: "2026-07-15T11:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:39Z" name: sequence-graph-30120-7b65c6b7f-q2rbc.18c271eba01a071e namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: 5ccbf1ca-72d4-4bf4-999a-10d5210e2b44 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30120} kind: Pod name: sequence-graph-30120-7b65c6b7f-q2rbc namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: c17d742e-9485-43cf-bf97-048f41f53351 kind: Event lastTimestamp: "2026-07-15T11:07:39Z" message: 'Created container: sequence-graph-30120' metadata: creationTimestamp: "2026-07-15T11:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:39Z" name: sequence-graph-30120-7b65c6b7f-q2rbc.18c271eba34ed72a namespace: kserve-ci-e2e-test resourceVersion: "20943" uid: 7b4d0825-253c-4318-8c2a-7ec2934f235b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30120} kind: Pod name: sequence-graph-30120-7b65c6b7f-q2rbc namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: c17d742e-9485-43cf-bf97-048f41f53351 kind: Event lastTimestamp: "2026-07-15T11:07:39Z" message: Started container sequence-graph-30120 metadata: creationTimestamp: "2026-07-15T11:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:39Z" name: sequence-graph-30120-7b65c6b7f-q2rbc.18c271eba3c3bea5 namespace: kserve-ci-e2e-test resourceVersion: "20944" uid: adfb9e6d-4b67-4765-a94b-51d6170f00af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30120} kind: Pod name: sequence-graph-30120-7b65c6b7f-q2rbc namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: c17d742e-9485-43cf-bf97-048f41f53351 kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: Stopping container sequence-graph-30120 metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:48Z" name: sequence-graph-30120-7b65c6b7f-q2rbc.18c271edb57ec4df namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 3f8feded-5faf-499d-be87-533ad0ae35ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30120} kind: Pod name: sequence-graph-30120-7b65c6b7f-q2rbc namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: c17d742e-9485-43cf-bf97-048f41f53351 kind: Event lastTimestamp: "2026-07-15T11:08:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-15T11:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:08:15Z" name: sequence-graph-30120-7b65c6b7f-q2rbc.18c271ee50d9124f namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: f9190d82-dccf-44ce-acab-57c489d59226 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-30120-7b65c6b7f namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: 4f9535c5-a328-41b0-8d34-675711e4b15b kind: Event lastTimestamp: "2026-07-15T11:07:38Z" message: 'Created pod: sequence-graph-30120-7b65c6b7f-q2rbc' metadata: creationTimestamp: "2026-07-15T11:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:38Z" name: sequence-graph-30120-7b65c6b7f.18c271eb5d79e2e8 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: a3f6e02e-40b2-49d0-a5e9-2d8c9c21dff4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-30120 namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: f8a89904-0ef5-4b3d-8ba0-efcc22d93856 kind: Event lastTimestamp: "2026-07-15T11:07:38Z" message: Scaled up replica set sequence-graph-30120-7b65c6b7f from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:38Z" name: sequence-graph-30120.18c271eb5c99c883 namespace: kserve-ci-e2e-test resourceVersion: "20905" uid: 09d87420-1fa8-4490-95ed-7508e4a3674f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-30120 namespace: kserve-ci-e2e-test resourceVersion: "20918" uid: 06db779c-871d-461c-ade7-db4598062d18 kind: Event lastTimestamp: "2026-07-15T11:07:38Z" message: InferenceGraph [sequence-graph-30120] is Ready metadata: creationTimestamp: "2026-07-15T11:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:07:38Z" name: sequence-graph-30120.18c271eb64e42466 namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: 2bb3be76-cabb-4bd2-bfb8-63e2af5d9a90 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:08:48.402880Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-70137-5dcc4d4f8-8llvh namespace: kserve-ci-e2e-test resourceVersion: "21713" uid: f8d4b7a8-b3b6-47d7-afda-c02c919415c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-70137-5dcc4d4f8-8llvh to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-07-15T11:08:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:08:48Z" name: sequence-graph-70137-5dcc4d4f8-8llvh.18c271fbbc5a1209 namespace: kserve-ci-e2e-test resourceVersion: "21718" uid: 034ba8e6-a509-448a-9cf6-3a601703a0b2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:48Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-70137-5dcc4d4f8-8llvh namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: f8d4b7a8-b3b6-47d7-afda-c02c919415c9 kind: Event lastTimestamp: "2026-07-15T11:08:48Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:08:48Z" name: sequence-graph-70137-5dcc4d4f8-8llvh.18c271fbd6a0dcc2 namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 3955e079-c208-4578-914d-447836315ea7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-70137} kind: Pod name: sequence-graph-70137-5dcc4d4f8-8llvh namespace: kserve-ci-e2e-test resourceVersion: "21715" uid: f8d4b7a8-b3b6-47d7-afda-c02c919415c9 kind: Event lastTimestamp: "2026-07-15T11:08:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0e1a169e211c4cdcd19b5d9ae2b04ed590c1bcb2f57424f5c6e7d0d0f2faa1f0" already present on machine metadata: creationTimestamp: "2026-07-15T11:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:08:48Z" name: sequence-graph-70137-5dcc4d4f8-8llvh.18c271fbd7fb0c82 namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: a0454373-997f-46b9-8b2a-317f689f74ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-70137} kind: Pod name: sequence-graph-70137-5dcc4d4f8-8llvh namespace: kserve-ci-e2e-test resourceVersion: "21715" uid: f8d4b7a8-b3b6-47d7-afda-c02c919415c9 kind: Event lastTimestamp: "2026-07-15T11:08:48Z" message: 'Created container: sequence-graph-70137' metadata: creationTimestamp: "2026-07-15T11:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:08:48Z" name: sequence-graph-70137-5dcc4d4f8-8llvh.18c271fbdb325f78 namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: c5cda6d0-3e7a-4096-8019-9b00a0baff39 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-70137} kind: Pod name: sequence-graph-70137-5dcc4d4f8-8llvh namespace: kserve-ci-e2e-test resourceVersion: "21715" uid: f8d4b7a8-b3b6-47d7-afda-c02c919415c9 kind: Event lastTimestamp: "2026-07-15T11:08:48Z" message: Started container sequence-graph-70137 metadata: creationTimestamp: "2026-07-15T11:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:08:48Z" name: sequence-graph-70137-5dcc4d4f8-8llvh.18c271fbdbabba1d namespace: kserve-ci-e2e-test resourceVersion: "21740" uid: f1c2d504-16f3-44d5-aee1-70ba007260d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-70137} kind: Pod name: sequence-graph-70137-5dcc4d4f8-8llvh namespace: kserve-ci-e2e-test resourceVersion: "21715" uid: f8d4b7a8-b3b6-47d7-afda-c02c919415c9 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Stopping container sequence-graph-70137 metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: sequence-graph-70137-5dcc4d4f8-8llvh.18c2726ee7d51174 namespace: kserve-ci-e2e-test resourceVersion: "25321" uid: 891b3189-3822-42a6-87f7-b705d905e881 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:17:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-70137} kind: Pod name: sequence-graph-70137-5dcc4d4f8-8llvh namespace: kserve-ci-e2e-test resourceVersion: "21715" uid: f8d4b7a8-b3b6-47d7-afda-c02c919415c9 kind: Event lastTimestamp: "2026-07-15T11:17:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-15T11:17:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:30Z" name: sequence-graph-70137-5dcc4d4f8-8llvh.18c2726f6132fe4e namespace: kserve-ci-e2e-test resourceVersion: "25878" uid: 6fc61a15-ab69-445c-ba1d-a89a7e36ce94 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-70137-5dcc4d4f8 namespace: kserve-ci-e2e-test resourceVersion: "21709" uid: 6764e72b-1178-4eed-a57b-43b84ca89b6b kind: Event lastTimestamp: "2026-07-15T11:08:48Z" message: 'Created pod: sequence-graph-70137-5dcc4d4f8-8llvh' metadata: creationTimestamp: "2026-07-15T11:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:48Z" name: sequence-graph-70137-5dcc4d4f8.18c271fbbbdd7808 namespace: kserve-ci-e2e-test resourceVersion: "21714" uid: 18f66d74-8906-4974-9110-8c31a94d3b99 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-70137 namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: 6b72382e-40c3-464c-8209-d85999e4bce7 kind: Event lastTimestamp: "2026-07-15T11:08:48Z" message: Scaled up replica set sequence-graph-70137-5dcc4d4f8 from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:48Z" name: sequence-graph-70137.18c271fbbaf3be21 namespace: kserve-ci-e2e-test resourceVersion: "21711" uid: bfe6e1d6-c4f4-462f-aeb3-cf5ca459c1d7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-70137 namespace: kserve-ci-e2e-test resourceVersion: "21726" uid: 7838375d-7018-49df-836d-d80cd410310d kind: Event lastTimestamp: "2026-07-15T11:08:48Z" message: InferenceGraph [sequence-graph-70137] is Ready metadata: creationTimestamp: "2026-07-15T11:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:08:48Z" name: sequence-graph-70137.18c271fbc033a51c namespace: kserve-ci-e2e-test resourceVersion: "21728" uid: 4bfe3de1-8fd0-472f-960b-90a36bccc5ca reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:09:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-70137 namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 8120d66d-b99a-4c11-80e0-5cec85dd59b7 kind: Event lastTimestamp: "2026-07-15T11:09:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:09:03Z" name: sequence-graph-70137.18c271ff3930fcaf namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: c4e29318-64b3-48a3-86a5-2484291506eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:09:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-70137 namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 8120d66d-b99a-4c11-80e0-5cec85dd59b7 kind: Event lastTimestamp: "2026-07-15T11:09:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:09:03Z" name: sequence-graph-70137.18c271ff39763fc4 namespace: kserve-ci-e2e-test resourceVersion: "21856" uid: e272570c-be60-45cb-b26f-6853f39cb0ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:09:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-70137 namespace: kserve-ci-e2e-test resourceVersion: "21855" uid: 8120d66d-b99a-4c11-80e0-5cec85dd59b7 kind: Event lastTimestamp: "2026-07-15T11:09:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:09:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:09:18Z" name: sequence-graph-70137.18c27202b8362514 namespace: kserve-ci-e2e-test resourceVersion: "21965" uid: 9212c197-09e9-4076-ac2e-527beb291567 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:09:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-70137 namespace: kserve-ci-e2e-test resourceVersion: "21855" uid: 8120d66d-b99a-4c11-80e0-5cec85dd59b7 kind: Event lastTimestamp: "2026-07-15T11:09:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:09:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:09:18Z" name: sequence-graph-70137.18c27202b8af074a namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: 6b7f20ad-77d4-4e81-8a68-d2d851a683e3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:17:23.191323Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-cdb7a-ffd5b4495-dstlk namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: a8cff83e-bc5d-458e-839a-a907efbf1670 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-cdb7a-ffd5b4495-dstlk to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-07-15T11:17:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:17:23Z" name: splitter-graph-cdb7a-ffd5b4495-dstlk.18c2727398224d80 namespace: kserve-ci-e2e-test resourceVersion: "25652" uid: 77deb228-c36a-4a07-b555-10883512b71a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:23Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-cdb7a-ffd5b4495-dstlk namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: a8cff83e-bc5d-458e-839a-a907efbf1670 kind: Event lastTimestamp: "2026-07-15T11:17:23Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:17:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:17:23Z" name: splitter-graph-cdb7a-ffd5b4495-dstlk.18c27273b23a140d namespace: kserve-ci-e2e-test resourceVersion: "25669" uid: 4197d3bb-1e29-4f23-b503-648f2b77883b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-cdb7a} kind: Pod name: splitter-graph-cdb7a-ffd5b4495-dstlk namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: a8cff83e-bc5d-458e-839a-a907efbf1670 kind: Event lastTimestamp: "2026-07-15T11:17:23Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0e1a169e211c4cdcd19b5d9ae2b04ed590c1bcb2f57424f5c6e7d0d0f2faa1f0" already present on machine metadata: creationTimestamp: "2026-07-15T11:17:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:23Z" name: splitter-graph-cdb7a-ffd5b4495-dstlk.18c27273b383d578 namespace: kserve-ci-e2e-test resourceVersion: "25671" uid: 9d862d86-9449-4304-b76f-2b06d3966a9d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-cdb7a} kind: Pod name: splitter-graph-cdb7a-ffd5b4495-dstlk namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: a8cff83e-bc5d-458e-839a-a907efbf1670 kind: Event lastTimestamp: "2026-07-15T11:17:23Z" message: 'Created container: splitter-graph-cdb7a' metadata: creationTimestamp: "2026-07-15T11:17:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:23Z" name: splitter-graph-cdb7a-ffd5b4495-dstlk.18c27273b67d64e9 namespace: kserve-ci-e2e-test resourceVersion: "25672" uid: c9c54efa-11bf-4d57-a999-957b61ebe759 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-cdb7a} kind: Pod name: splitter-graph-cdb7a-ffd5b4495-dstlk namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: a8cff83e-bc5d-458e-839a-a907efbf1670 kind: Event lastTimestamp: "2026-07-15T11:17:23Z" message: Started container splitter-graph-cdb7a metadata: creationTimestamp: "2026-07-15T11:17:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:23Z" name: splitter-graph-cdb7a-ffd5b4495-dstlk.18c27273b6f7bc03 namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: 91698e25-4b7b-4bec-a42a-b771ad06d534 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-cdb7a} kind: Pod name: splitter-graph-cdb7a-ffd5b4495-dstlk namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: a8cff83e-bc5d-458e-839a-a907efbf1670 kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: Stopping container splitter-graph-cdb7a metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:33Z" name: splitter-graph-cdb7a-ffd5b4495-dstlk.18c27275f12850af namespace: kserve-ci-e2e-test resourceVersion: "26066" uid: d8894e88-48fa-44cf-be88-883315ef2438 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:17:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-cdb7a} kind: Pod name: splitter-graph-cdb7a-ffd5b4495-dstlk namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: a8cff83e-bc5d-458e-839a-a907efbf1670 kind: Event lastTimestamp: "2026-07-15T11:18:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-15T11:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:18:00Z" name: splitter-graph-cdb7a-ffd5b4495-dstlk.18c27276758fe2a6 namespace: kserve-ci-e2e-test resourceVersion: "26428" uid: 70584f67-37c4-4c6a-928b-c20390702435 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-cdb7a-ffd5b4495 namespace: kserve-ci-e2e-test resourceVersion: "25644" uid: d8091436-bdde-449b-8545-e48f84537c27 kind: Event lastTimestamp: "2026-07-15T11:17:23Z" message: 'Created pod: splitter-graph-cdb7a-ffd5b4495-dstlk' metadata: creationTimestamp: "2026-07-15T11:17:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:23Z" name: splitter-graph-cdb7a-ffd5b4495.18c2727397a896be namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: 81876e06-4477-43f9-b0a0-359c91dc5ad4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-cdb7a namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 2664cdd3-acbd-4caf-8a57-08457d99c12d kind: Event lastTimestamp: "2026-07-15T11:17:23Z" message: Scaled up replica set splitter-graph-cdb7a-ffd5b4495 from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:17:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:23Z" name: splitter-graph-cdb7a.18c2727396cc06de namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: 97697c0a-fb2f-4942-9788-9999934d4dff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-cdb7a namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 7db2d797-6b78-40f2-b9be-c3019f0afcb5 kind: Event lastTimestamp: "2026-07-15T11:17:23Z" message: InferenceGraph [splitter-graph-cdb7a] is Ready metadata: creationTimestamp: "2026-07-15T11:17:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:23Z" name: splitter-graph-cdb7a.18c272739cf5be42 namespace: kserve-ci-e2e-test resourceVersion: "25662" uid: d9d72b37-124b-4f9a-8f8f-47753f324cd1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:18:33.521664Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-d1ffd-7d9bf8b7bd-962wv namespace: kserve-ci-e2e-test resourceVersion: "26766" uid: dc02e89b-ee61-4f1d-8c0c-8b69665e24ec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-d1ffd-7d9bf8b7bd-962wv to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-07-15T11:18:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:18:33Z" name: splitter-graph-d1ffd-7d9bf8b7bd-962wv.18c27283f8262d08 namespace: kserve-ci-e2e-test resourceVersion: "26771" uid: 4ac00ffa-db64-44e5-a5f0-ab715c1be205 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:33Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-d1ffd-7d9bf8b7bd-962wv namespace: kserve-ci-e2e-test resourceVersion: "26775" uid: dc02e89b-ee61-4f1d-8c0c-8b69665e24ec kind: Event lastTimestamp: "2026-07-15T11:18:33Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:18:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:18:33Z" name: splitter-graph-d1ffd-7d9bf8b7bd-962wv.18c272841270f3b0 namespace: kserve-ci-e2e-test resourceVersion: "26790" uid: c36425dc-5c32-4194-aea5-3787d0aba950 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d1ffd} kind: Pod name: splitter-graph-d1ffd-7d9bf8b7bd-962wv namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: dc02e89b-ee61-4f1d-8c0c-8b69665e24ec kind: Event lastTimestamp: "2026-07-15T11:18:33Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0e1a169e211c4cdcd19b5d9ae2b04ed590c1bcb2f57424f5c6e7d0d0f2faa1f0" already present on machine metadata: creationTimestamp: "2026-07-15T11:18:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:18:33Z" name: splitter-graph-d1ffd-7d9bf8b7bd-962wv.18c2728413fa116b namespace: kserve-ci-e2e-test resourceVersion: "26792" uid: 020c000a-1532-4870-8975-9dfe6a392e97 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d1ffd} kind: Pod name: splitter-graph-d1ffd-7d9bf8b7bd-962wv namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: dc02e89b-ee61-4f1d-8c0c-8b69665e24ec kind: Event lastTimestamp: "2026-07-15T11:18:34Z" message: 'Created container: splitter-graph-d1ffd' metadata: creationTimestamp: "2026-07-15T11:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:18:34Z" name: splitter-graph-d1ffd-7d9bf8b7bd-962wv.18c27284180f8fb3 namespace: kserve-ci-e2e-test resourceVersion: "26794" uid: d3bcb130-73df-43a3-9942-a708183a1658 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d1ffd} kind: Pod name: splitter-graph-d1ffd-7d9bf8b7bd-962wv namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: dc02e89b-ee61-4f1d-8c0c-8b69665e24ec kind: Event lastTimestamp: "2026-07-15T11:18:34Z" message: Started container splitter-graph-d1ffd metadata: creationTimestamp: "2026-07-15T11:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:18:34Z" name: splitter-graph-d1ffd-7d9bf8b7bd-962wv.18c27284188b599b namespace: kserve-ci-e2e-test resourceVersion: "26795" uid: 879ab239-8d44-469c-9bba-c8bd75542111 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:26:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d1ffd} kind: Pod name: splitter-graph-d1ffd-7d9bf8b7bd-962wv namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: dc02e89b-ee61-4f1d-8c0c-8b69665e24ec kind: Event lastTimestamp: "2026-07-15T11:26:48Z" message: Stopping container splitter-graph-d1ffd metadata: creationTimestamp: "2026-07-15T11:26:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:26:48Z" name: splitter-graph-d1ffd-7d9bf8b7bd-962wv.18c272f725d9643c namespace: kserve-ci-e2e-test resourceVersion: "30205" uid: f9953239-de73-4e51-a1da-a5d2ce413b94 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:26:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-d1ffd} kind: Pod name: splitter-graph-d1ffd-7d9bf8b7bd-962wv namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: dc02e89b-ee61-4f1d-8c0c-8b69665e24ec kind: Event lastTimestamp: "2026-07-15T11:27:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-15T11:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:27:15Z" name: splitter-graph-d1ffd-7d9bf8b7bd-962wv.18c272f7c2382e2e namespace: kserve-ci-e2e-test resourceVersion: "30443" uid: 33b14b88-f947-4455-b62e-78e89dad6e2a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-d1ffd-7d9bf8b7bd namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: 5effe28f-7ace-4655-a743-85145c9928bf kind: Event lastTimestamp: "2026-07-15T11:18:33Z" message: 'Created pod: splitter-graph-d1ffd-7d9bf8b7bd-962wv' metadata: creationTimestamp: "2026-07-15T11:18:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:33Z" name: splitter-graph-d1ffd-7d9bf8b7bd.18c27283f7b623ff namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: a4a8ef11-0a8a-4175-a552-66f5054f5dc8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-d1ffd namespace: kserve-ci-e2e-test resourceVersion: "26759" uid: 50287802-d971-4ec7-b154-c59af7544249 kind: Event lastTimestamp: "2026-07-15T11:18:33Z" message: Scaled up replica set splitter-graph-d1ffd-7d9bf8b7bd from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:18:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:33Z" name: splitter-graph-d1ffd.18c27283f68873d7 namespace: kserve-ci-e2e-test resourceVersion: "26764" uid: 436ed072-3a80-4b05-9a8f-44af2b3cf5aa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-d1ffd namespace: kserve-ci-e2e-test resourceVersion: "26780" uid: cfaf2077-efd5-4c19-b23d-21258308add5 kind: Event lastTimestamp: "2026-07-15T11:18:33Z" message: InferenceGraph [splitter-graph-d1ffd] is Ready metadata: creationTimestamp: "2026-07-15T11:18:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:18:33Z" name: splitter-graph-d1ffd.18c27283fc395584 namespace: kserve-ci-e2e-test resourceVersion: "26783" uid: fdb1ea92-b550-4b8b-ade1-b12a4fd1e279 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:18:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-d1ffd namespace: kserve-ci-e2e-test resourceVersion: "26761" uid: b001d66f-2f03-4abe-a4a8-6c61e98955b2 kind: Event lastTimestamp: "2026-07-15T11:19:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:18:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:19:03Z" name: splitter-graph-d1ffd.18c2728775022535 namespace: kserve-ci-e2e-test resourceVersion: "27006" uid: d4f12245-1a05-4343-94df-2412eae274c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:18:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-d1ffd namespace: kserve-ci-e2e-test resourceVersion: "26761" uid: b001d66f-2f03-4abe-a4a8-6c61e98955b2 kind: Event lastTimestamp: "2026-07-15T11:19:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:18:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:19:03Z" name: splitter-graph-d1ffd.18c272877567a195 namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 78046630-854a-4579-a20c-c87d59a6fffb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-15T10:57:13.092755Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: 582d624f-7f4f-459e-af00-7e5555fe81fd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T10:57:13Z" name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp.18c27159d8a384f2 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 9921b0c1-c589-47a9-bd16-301433c8b0a2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: 582d624f-7f4f-459e-af00-7e5555fe81fd kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T10:57:13Z" name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp.18c27159f20931cc namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: d83be981-4d26-4561-bf91-9be66fe2c455 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 582d624f-7f4f-459e-af00-7e5555fe81fd kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp.18c27159f356be46 namespace: kserve-ci-e2e-test resourceVersion: "15793" uid: 24ec301e-f676-499d-a055-f1f9a9c460e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 582d624f-7f4f-459e-af00-7e5555fe81fd kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp.18c27159f66cbb1c namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: 4158567d-a5ca-4227-ac15-db8621a02169 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 582d624f-7f4f-459e-af00-7e5555fe81fd kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp.18c27159f6e5dbb8 namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: 863881e9-87ba-4100-9251-4d093dfda1dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 582d624f-7f4f-459e-af00-7e5555fe81fd kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp.18c27159f6f352df namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: b65e397a-e23e-436c-ba8d-0c536a41ca0e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 582d624f-7f4f-459e-af00-7e5555fe81fd kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp.18c27159faba1169 namespace: kserve-ci-e2e-test resourceVersion: "15800" uid: 867a51b9-dba7-4ae8-ae12-7c968cafc5a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 582d624f-7f4f-459e-af00-7e5555fe81fd kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:13Z" name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp.18c27159fb24dc3f namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: dc61dc42-4308-41d2-b890-e1a2f5c9e0b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-15T10:57:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 582d624f-7f4f-459e-af00-7e5555fe81fd kind: Event lastTimestamp: "2026-07-15T11:06:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:31Z" name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp.18c2715a6703463b namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 930f72c0-bd8f-4647-807c-175bbe3e256c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 582d624f-7f4f-459e-af00-7e5555fe81fd kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:28Z" name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp.18c271db145bba4e namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: 4e0825c8-b479-4acd-a583-938de89e363b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 582d624f-7f4f-459e-af00-7e5555fe81fd kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:28Z" name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp.18c271db145c57a7 namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 80c04abb-c5e9-42bf-a608-829a1b4cc478 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 582d624f-7f4f-459e-af00-7e5555fe81fd kind: Event lastTimestamp: "2026-07-15T11:06:31Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:31Z" name: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp.18c271dbdb8a8ac7 namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 8385b752-7128-44e8-9997-076558e8e388 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-074d2-predictor-6f6bcf5df6 namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: d8ea6abb-8c2c-4571-8a4a-2b12b3caeb25 kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: 'Created pod: success-200-isvc-074d2-predictor-6f6bcf5df6-7bnmp' metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:13Z" name: success-200-isvc-074d2-predictor-6f6bcf5df6.18c27159d81d3549 namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: 28de7003-2c88-48f3-bb59-358b733994f1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-074d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: 8346c542-2d08-4e35-81c9-e0ab081839db kind: Event lastTimestamp: "2026-07-15T10:57:13Z" message: Scaled up replica set success-200-isvc-074d2-predictor-6f6bcf5df6 from 0 to 1 metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:13Z" name: success-200-isvc-074d2-predictor.18c27159d5f9ad74 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: e2028d77-6b3e-4a98-a94d-7966e32b241f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T10:57:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-074d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: 76dc2b74-47f0-4181-8d39-791dc8bc322e kind: Event lastTimestamp: "2026-07-15T10:57:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:57:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:43Z" name: success-200-isvc-074d2-predictor.18c2715d547d7d55 namespace: kserve-ci-e2e-test resourceVersion: "16122" uid: e45fb89a-37f7-4136-af15-21937b340eae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T10:57:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-074d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: 76dc2b74-47f0-4181-8d39-791dc8bc322e kind: Event lastTimestamp: "2026-07-15T10:57:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:57:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:43Z" name: success-200-isvc-074d2-predictor.18c2715d54fd9fdb namespace: kserve-ci-e2e-test resourceVersion: "16123" uid: d617314e-d6d5-40a0-bdbc-c67ec3d63078 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-15T10:57:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-074d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: 76dc2b74-47f0-4181-8d39-791dc8bc322e kind: Event lastTimestamp: "2026-07-15T10:58:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:58:43Z" name: success-200-isvc-074d2-predictor.18c2716452f19fcf namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: d146c5cf-6431-44be-a67b-c67d326f6516 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-15T10:57:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-074d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: 76dc2b74-47f0-4181-8d39-791dc8bc322e kind: Event lastTimestamp: "2026-07-15T10:58:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:58:43Z" name: success-200-isvc-074d2-predictor.18c27164535fc6eb namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: 970a2002-60e7-4833-8f75-a3f856a8b695 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T10:57:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-074d2 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 507868dd-a224-490c-bc25-847589f54d78 kind: Event lastTimestamp: "2026-07-15T10:58:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T10:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:58:01Z" name: success-200-isvc-074d2.18c27159d61860bf namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: c48407d1-8171-4053-88f8-897fed002dc8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:58:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-074d2 namespace: kserve-ci-e2e-test resourceVersion: "16286" uid: 507868dd-a224-490c-bc25-847589f54d78 kind: Event lastTimestamp: "2026-07-15T10:58:01Z" message: InferenceService [success-200-isvc-074d2] is Ready metadata: creationTimestamp: "2026-07-15T10:58:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:58:01Z" name: success-200-isvc-074d2.18c271652214ad11 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: f858d4fe-a097-4062-854f-09a56b8e2cc2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:06:28.153982Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: 89161cbe-9df8-4661-a724-8cef68e2fe35 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-30120-predictor-74997b5b5d-6hgnc to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:06:28Z" name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc.18c271db14ddb5c6 namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: 47a673a3-8783-471a-94f2-ee0097f1eb99 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 89161cbe-9df8-4661-a724-8cef68e2fe35 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-30120-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:28Z" name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc.18c271db1ea2c079 namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 01cbd727-1988-481e-bd9b-b0def2ae38ce reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 89161cbe-9df8-4661-a724-8cef68e2fe35 kind: Event lastTimestamp: "2026-07-15T11:06:29Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:06:29Z" name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc.18c271db52cba397 namespace: kserve-ci-e2e-test resourceVersion: "20096" uid: bc81783e-7642-421c-8288-6af326cd2211 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 89161cbe-9df8-4661-a724-8cef68e2fe35 kind: Event lastTimestamp: "2026-07-15T11:06:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:29Z" name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc.18c271db55020ab6 namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: bba7a397-4d4f-42ff-85d1-78b9c6089c1d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 89161cbe-9df8-4661-a724-8cef68e2fe35 kind: Event lastTimestamp: "2026-07-15T11:06:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:29Z" name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc.18c271db57e99b9e namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: 3814b631-27b9-4704-892d-594874e7010d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 89161cbe-9df8-4661-a724-8cef68e2fe35 kind: Event lastTimestamp: "2026-07-15T11:06:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:29Z" name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc.18c271db585b77d7 namespace: kserve-ci-e2e-test resourceVersion: "20100" uid: fe63c196-7075-4020-9f7f-dc4ecb9a2d21 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 89161cbe-9df8-4661-a724-8cef68e2fe35 kind: Event lastTimestamp: "2026-07-15T11:06:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:29Z" name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc.18c271db5864afa3 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 24399fca-f6ff-45f0-993c-0111b7f00713 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 89161cbe-9df8-4661-a724-8cef68e2fe35 kind: Event lastTimestamp: "2026-07-15T11:06:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:29Z" name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc.18c271db5b82afd1 namespace: kserve-ci-e2e-test resourceVersion: "20102" uid: 54c67570-c858-4820-9caf-05596b9f1119 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 89161cbe-9df8-4661-a724-8cef68e2fe35 kind: Event lastTimestamp: "2026-07-15T11:06:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:29Z" name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc.18c271db5bf2e1e8 namespace: kserve-ci-e2e-test resourceVersion: "20103" uid: 124b677a-08fb-4d2d-9849-cc80b813df44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-15T11:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 89161cbe-9df8-4661-a724-8cef68e2fe35 kind: Event lastTimestamp: "2026-07-15T11:07:16Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:16Z" name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc.18c271dbaa53d0d9 namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: f4740b7d-4dc1-4236-8f15-42f6748f4a3a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 89161cbe-9df8-4661-a724-8cef68e2fe35 kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:48Z" name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc.18c271edbceca28e namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 576b9b98-b1bc-4f29-93a6-5678559cd365 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 89161cbe-9df8-4661-a724-8cef68e2fe35 kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:48Z" name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc.18c271edbced1631 namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: 0a2de11b-3974-4f26-99af-6ed59ab8a270 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 89161cbe-9df8-4661-a724-8cef68e2fe35 kind: Event lastTimestamp: "2026-07-15T11:07:51Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:51Z" name: success-200-isvc-30120-predictor-74997b5b5d-6hgnc.18c271ee86b9c8da namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: 76691f66-1e51-4abc-99e2-4880c4136d02 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-30120-predictor-74997b5b5d namespace: kserve-ci-e2e-test resourceVersion: "20000" uid: fbeb91ea-2dbf-4a37-8b09-5b37b5623c4b kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: 'Created pod: success-200-isvc-30120-predictor-74997b5b5d-6hgnc' metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:28Z" name: success-200-isvc-30120-predictor-74997b5b5d.18c271db143d9f04 namespace: kserve-ci-e2e-test resourceVersion: "20022" uid: 4e034947-83b5-4acc-ba0f-2ef7c9c43eee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-30120-predictor namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: 6e942611-fd95-4092-b848-4ae715441bc7 kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: Scaled up replica set success-200-isvc-30120-predictor-74997b5b5d from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:28Z" name: success-200-isvc-30120-predictor.18c271db0f230297 namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: c83e62e0-bce0-40dc-904b-72d18d60da7b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:06:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-30120-predictor namespace: kserve-ci-e2e-test resourceVersion: "19996" uid: 58c7722d-e0cb-49cf-84ee-c383af656ef5 kind: Event lastTimestamp: "2026-07-15T11:06:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:58Z" name: success-200-isvc-30120-predictor.18c271de8df1cbf3 namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: a0706f7c-cfa7-4620-8ac9-bbf8bfe4876f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:06:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-30120-predictor namespace: kserve-ci-e2e-test resourceVersion: "19996" uid: 58c7722d-e0cb-49cf-84ee-c383af656ef5 kind: Event lastTimestamp: "2026-07-15T11:06:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:58Z" name: success-200-isvc-30120-predictor.18c271de8e590129 namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: aa09a1d3-13ed-4b5c-8dac-e6c9ea7f31cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:07:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-30120-predictor namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: 58c7722d-e0cb-49cf-84ee-c383af656ef5 kind: Event lastTimestamp: "2026-07-15T11:07:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:43Z" name: success-200-isvc-30120-predictor.18c271e58cc6a453 namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: f4d4f0af-718b-42e1-9649-6c7bd82c9a09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:07:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-30120-predictor namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: 58c7722d-e0cb-49cf-84ee-c383af656ef5 kind: Event lastTimestamp: "2026-07-15T11:07:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:43Z" name: success-200-isvc-30120-predictor.18c271e58d86c18b namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: 2eca68ce-56d0-4343-8b76-e8046a2abc13 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-30120 namespace: kserve-ci-e2e-test resourceVersion: "19987" uid: 835e097f-91fc-45db-944b-c207ed5fcc3e kind: Event lastTimestamp: "2026-07-15T11:07:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:07:26Z" name: success-200-isvc-30120.18c271db0f4a2b51 namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: 8fd3301f-b456-44a6-bfed-3e241d958a27 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-30120 namespace: kserve-ci-e2e-test resourceVersion: "19987" uid: 835e097f-91fc-45db-944b-c207ed5fcc3e kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: 'Failed to update status for InferenceService "success-200-isvc-30120": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-30120": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:06:28Z" name: success-200-isvc-30120.18c271db105cfdb7 namespace: kserve-ci-e2e-test resourceVersion: "20012" uid: 49dc103c-14a1-4fb1-815d-09afb808de67 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-30120 namespace: kserve-ci-e2e-test resourceVersion: "19987" uid: 835e097f-91fc-45db-944b-c207ed5fcc3e kind: Event lastTimestamp: "2026-07-15T11:06:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-30120": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:06:28Z" name: success-200-isvc-30120.18c271db105d6ad0 namespace: kserve-ci-e2e-test resourceVersion: "20014" uid: eab95915-0d9f-4a3a-9a93-7cdd114d4d79 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-30120 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: 835e097f-91fc-45db-944b-c207ed5fcc3e kind: Event lastTimestamp: "2026-07-15T11:07:26Z" message: InferenceService [success-200-isvc-30120] is Ready metadata: creationTimestamp: "2026-07-15T11:07:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:07:26Z" name: success-200-isvc-30120.18c271e8bab161da namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 1a2a91f1-943f-494e-b990-8d6a4909657c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T10:54:42.786867Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13958" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6db9e-predictor-58b664446b-fmthr to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T10:54:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T10:54:42Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c27136d9b5b6e8 namespace: kserve-ci-e2e-test resourceVersion: "13967" uid: db1eef3e-f989-4f09-9ae5-a0a57a7fcdaf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T10:54:43Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c27136f3abacda namespace: kserve-ci-e2e-test resourceVersion: "14014" uid: 72c188ac-9674-49f4-a58f-239a9c2732ce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: "2026-07-15T10:54:43Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1738" metadata: creationTimestamp: "2026-07-15T10:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:54:43Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c27136f6a3ab0e namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: 67a4d1f4-ddd1-42d9-ad59-28cbee3ee5a9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1738" in 18.934s (18.934s including waiting). Image size: 1335613541 bytes.' metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c2713b5f3bc6bb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c86afe95-b26d-4d8a-a0dd-e8a8b4fda232 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c2713b629d6cf5 namespace: kserve-ci-e2e-test resourceVersion: "14294" uid: 5923e050-096c-45fe-b844-001f19709195 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c2713b6317c54f namespace: kserve-ci-e2e-test resourceVersion: "14296" uid: 8a4ff868-4961-45b8-b022-1276d5c3950b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: "2026-07-15T10:55:02Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-15T10:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:02Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c2713b63272937 namespace: kserve-ci-e2e-test resourceVersion: "14297" uid: 357d829b-813c-4a4a-b473-6a0a90aea7da reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: "2026-07-15T10:55:07Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 5.058s (5.058s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:07Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c2713c90afbf9e namespace: kserve-ci-e2e-test resourceVersion: "14342" uid: 5e424d92-20d8-4720-902f-4120e03289d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: "2026-07-15T10:55:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:07Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c2713c9d4e1de4 namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: d892a730-d559-47c1-a53a-5117a764e484 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: "2026-07-15T10:55:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:55:07Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c2713c9eff7dbe namespace: kserve-ci-e2e-test resourceVersion: "14348" uid: ac89e74f-f7c5-48d9-ac45-7135886f3280 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-15T10:55:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:55:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c2713cb447f09d namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 3d48f7a5-ddab-41c7-ba6d-ac9346c67366 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c2715089d5c73d namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: 9b41d661-2d78-4361-b97b-75bd88cc8cee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c2715089d62130 namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 8b9943ac-a5ce-4a0e-bce5-ddb2bfa40658 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6db9e-predictor-58b664446b-fmthr namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: e9436c84-deca-4753-a2be-5ec45980638f kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-6db9e-predictor-58b664446b-fmthr.18c27150baa05ccf namespace: kserve-ci-e2e-test resourceVersion: "15277" uid: 5f78c9a4-4a18-42ba-ae25-f57118479f13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6db9e-predictor-58b664446b namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: f8a31ce8-086c-4d60-aad1-a2fc7b2f076c kind: Event lastTimestamp: "2026-07-15T10:54:42Z" message: 'Created pod: success-200-isvc-6db9e-predictor-58b664446b-fmthr' metadata: creationTimestamp: "2026-07-15T10:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:54:42Z" name: success-200-isvc-6db9e-predictor-58b664446b.18c27136d846080d namespace: kserve-ci-e2e-test resourceVersion: "13959" uid: 657a91da-2d55-4a4d-ad17-649370926154 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6db9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13945" uid: 8754d2b2-57d0-4590-af87-d9ec58264015 kind: Event lastTimestamp: "2026-07-15T10:54:42Z" message: Scaled up replica set success-200-isvc-6db9e-predictor-58b664446b from 0 to 1 metadata: creationTimestamp: "2026-07-15T10:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:54:42Z" name: success-200-isvc-6db9e-predictor.18c27136d072c9b5 namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: 15b5bfde-1565-468b-b151-86c3a4168972 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T10:54:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6db9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13954" uid: c9ebe499-c62e-49c5-bb1e-08f6d1712c4d kind: Event lastTimestamp: "2026-07-15T10:55:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:55:12Z" name: success-200-isvc-6db9e-predictor.18c2713a56557b54 namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: 7c786d5b-529e-47eb-9a7d-2f9e313e65b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T10:54:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6db9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13954" uid: c9ebe499-c62e-49c5-bb1e-08f6d1712c4d kind: Event lastTimestamp: "2026-07-15T10:55:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:55:12Z" name: success-200-isvc-6db9e-predictor.18c2713a56b9d26b namespace: kserve-ci-e2e-test resourceVersion: "14408" uid: f02f6d2a-7667-4226-8da9-9ae8e3641ba0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T10:55:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6db9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: c9ebe499-c62e-49c5-bb1e-08f6d1712c4d kind: Event lastTimestamp: "2026-07-15T10:56:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:27Z" name: success-200-isvc-6db9e-predictor.18c2714155c6f2cc namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: d904350f-cfb5-4805-a409-aca5f0db625b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T10:55:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6db9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: c9ebe499-c62e-49c5-bb1e-08f6d1712c4d kind: Event lastTimestamp: "2026-07-15T10:56:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:27Z" name: success-200-isvc-6db9e-predictor.18c27141563a4a62 namespace: kserve-ci-e2e-test resourceVersion: "15080" uid: 3b140351-3dd6-4a00-88f6-9891d7f68008 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T10:54:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6db9e namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: 446ba854-589e-42ba-8f3d-ab88396a904e kind: Event lastTimestamp: "2026-07-15T10:56:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T10:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:56:04Z" name: success-200-isvc-6db9e.18c27136d7333461 namespace: kserve-ci-e2e-test resourceVersion: "14855" uid: 08940a9c-6db8-4861-ad18-039f4c81f54c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6db9e namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: 446ba854-589e-42ba-8f3d-ab88396a904e kind: Event lastTimestamp: "2026-07-15T10:54:42Z" message: 'Failed to update status for InferenceService "success-200-isvc-6db9e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6db9e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T10:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:54:42Z" name: success-200-isvc-6db9e.18c27136d8015145 namespace: kserve-ci-e2e-test resourceVersion: "13960" uid: 8e6ebb4f-b3bf-42fd-9b0f-de9d81041854 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:54:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6db9e namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: 446ba854-589e-42ba-8f3d-ab88396a904e kind: Event lastTimestamp: "2026-07-15T10:54:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6db9e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T10:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:54:42Z" name: success-200-isvc-6db9e.18c27136d801b027 namespace: kserve-ci-e2e-test resourceVersion: "13964" uid: 463e9fc1-cfdc-493e-b5a1-ff832e68534f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6db9e namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: 446ba854-589e-42ba-8f3d-ab88396a904e kind: Event lastTimestamp: "2026-07-15T10:56:04Z" message: InferenceService [success-200-isvc-6db9e] is Ready metadata: creationTimestamp: "2026-07-15T10:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:56:04Z" name: success-200-isvc-6db9e.18c27149c4f5cd97 namespace: kserve-ci-e2e-test resourceVersion: "14854" uid: f61dadb0-5edb-4716-bb33-21ae567de068 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:07:48.366941Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-70137-predictor-77585cb866-kcdrt namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: 0d80ad20-8011-4e76-a4d9-2d8335b4e25c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-70137-predictor-77585cb866-kcdrt to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:07:48Z" name: success-200-isvc-70137-predictor-77585cb866-kcdrt.18c271edc1ee4fb8 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: 5ad05db6-57b8-492a-9f97-0e5caca3eb14 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-70137-predictor-77585cb866-kcdrt namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 0d80ad20-8011-4e76-a4d9-2d8335b4e25c kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-70137-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:48Z" name: success-200-isvc-70137-predictor-77585cb866-kcdrt.18c271edcf6c0ff7 namespace: kserve-ci-e2e-test resourceVersion: "21123" uid: 390010c0-1656-46eb-859c-6fcccb64e9ca reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-70137-predictor-77585cb866-kcdrt namespace: kserve-ci-e2e-test resourceVersion: "21096" uid: 0d80ad20-8011-4e76-a4d9-2d8335b4e25c kind: Event lastTimestamp: "2026-07-15T11:07:49Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:07:49Z" name: success-200-isvc-70137-predictor-77585cb866-kcdrt.18c271edffbd49b9 namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: 2e5f5b4b-0709-4640-b8c5-b5694eceeacf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-70137-predictor-77585cb866-kcdrt namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 0d80ad20-8011-4e76-a4d9-2d8335b4e25c kind: Event lastTimestamp: "2026-07-15T11:07:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:49Z" name: success-200-isvc-70137-predictor-77585cb866-kcdrt.18c271ee0124c484 namespace: kserve-ci-e2e-test resourceVersion: "21150" uid: 1245de99-4905-410d-8eee-94b9d4be4987 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-70137-predictor-77585cb866-kcdrt namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 0d80ad20-8011-4e76-a4d9-2d8335b4e25c kind: Event lastTimestamp: "2026-07-15T11:07:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:49Z" name: success-200-isvc-70137-predictor-77585cb866-kcdrt.18c271ee04343056 namespace: kserve-ci-e2e-test resourceVersion: "21151" uid: 98dec1a9-5505-4308-a8f3-faf26c220f6f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-70137-predictor-77585cb866-kcdrt namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 0d80ad20-8011-4e76-a4d9-2d8335b4e25c kind: Event lastTimestamp: "2026-07-15T11:07:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:49Z" name: success-200-isvc-70137-predictor-77585cb866-kcdrt.18c271ee04a2f713 namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: ce68adb7-eef4-48b2-bb65-2fbc2b63657e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-70137-predictor-77585cb866-kcdrt namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 0d80ad20-8011-4e76-a4d9-2d8335b4e25c kind: Event lastTimestamp: "2026-07-15T11:07:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:49Z" name: success-200-isvc-70137-predictor-77585cb866-kcdrt.18c271ee04acbe36 namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 1e481037-472f-479b-913b-cdec1a603ce3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-70137-predictor-77585cb866-kcdrt namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 0d80ad20-8011-4e76-a4d9-2d8335b4e25c kind: Event lastTimestamp: "2026-07-15T11:07:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:49Z" name: success-200-isvc-70137-predictor-77585cb866-kcdrt.18c271ee07ffde9f namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: 244c9072-706b-48a7-adeb-d33ad06e1088 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-70137-predictor-77585cb866-kcdrt namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 0d80ad20-8011-4e76-a4d9-2d8335b4e25c kind: Event lastTimestamp: "2026-07-15T11:07:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:49Z" name: success-200-isvc-70137-predictor-77585cb866-kcdrt.18c271ee086fc04e namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: 3280c0c8-0028-4c07-a78e-e539cbb36444 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-15T11:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-70137-predictor-77585cb866-kcdrt namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 0d80ad20-8011-4e76-a4d9-2d8335b4e25c kind: Event lastTimestamp: "2026-07-15T11:17:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:07Z" name: success-200-isvc-70137-predictor-77585cb866-kcdrt.18c271ee5feb8057 namespace: kserve-ci-e2e-test resourceVersion: "25477" uid: 849961d0-e4b3-4a4d-8f6c-57b105bafeb4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-70137-predictor-77585cb866-kcdrt namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 0d80ad20-8011-4e76-a4d9-2d8335b4e25c kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-70137-predictor-77585cb866-kcdrt.18c2726eef68adce namespace: kserve-ci-e2e-test resourceVersion: "25361" uid: 6022f19d-0a4e-45ba-af5f-51a1f12ce559 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-70137-predictor-77585cb866-kcdrt namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 0d80ad20-8011-4e76-a4d9-2d8335b4e25c kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-70137-predictor-77585cb866-kcdrt.18c2726eef69219b namespace: kserve-ci-e2e-test resourceVersion: "25364" uid: ef7221a3-1ace-48c5-a8e4-310f776e8721 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-70137-predictor-77585cb866-kcdrt namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 0d80ad20-8011-4e76-a4d9-2d8335b4e25c kind: Event lastTimestamp: "2026-07-15T11:17:07Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:07Z" name: success-200-isvc-70137-predictor-77585cb866-kcdrt.18c2726fd48b022d namespace: kserve-ci-e2e-test resourceVersion: "25474" uid: 311036a4-1d06-4de0-bca4-c037b26cf2e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-70137-predictor-77585cb866 namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: e49d148e-377b-4956-ad79-899920130ee5 kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: 'Created pod: success-200-isvc-70137-predictor-77585cb866-kcdrt' metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:48Z" name: success-200-isvc-70137-predictor-77585cb866.18c271edc16ad687 namespace: kserve-ci-e2e-test resourceVersion: "21082" uid: 67dc6dde-bf99-4932-8071-e96211eab648 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-70137-predictor namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: a645c584-8c80-4df8-ad32-c5505ccdeef6 kind: Event lastTimestamp: "2026-07-15T11:07:48Z" message: Scaled up replica set success-200-isvc-70137-predictor-77585cb866 from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:48Z" name: success-200-isvc-70137-predictor.18c271edbdee4cbb namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: 2d9defcc-fe41-44f9-ab52-82721416913d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-70137-predictor namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: 2106e047-f214-45ae-a8de-f47d92a82cce kind: Event lastTimestamp: "2026-07-15T11:08:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:03Z" name: success-200-isvc-70137-predictor.18c271f13ca2bdd7 namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: 060ddd18-58c8-484a-8bd8-dfd6f8f869fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-70137-predictor namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: 2106e047-f214-45ae-a8de-f47d92a82cce kind: Event lastTimestamp: "2026-07-15T11:08:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:03Z" name: success-200-isvc-70137-predictor.18c271f13d0b2bcd namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: 0715576d-9d37-40bb-b9d5-b4ec2c2f8f89 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:08:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-70137-predictor namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 2106e047-f214-45ae-a8de-f47d92a82cce kind: Event lastTimestamp: "2026-07-15T11:09:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:08:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:09:18Z" name: success-200-isvc-70137-predictor.18c271f4bbce1bd5 namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: ca7ab2c0-67e1-47f4-83a9-0e35926b48bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:08:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-70137-predictor namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 2106e047-f214-45ae-a8de-f47d92a82cce kind: Event lastTimestamp: "2026-07-15T11:09:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:08:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:09:18Z" name: success-200-isvc-70137-predictor.18c271f4bc3a6ec6 namespace: kserve-ci-e2e-test resourceVersion: "21964" uid: f4f78dc5-0101-4852-8d20-d5cd8fd643f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:07:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-70137 namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 37c02846-39e8-4700-a596-b0ef40e995d3 kind: Event lastTimestamp: "2026-07-15T11:08:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:08:37Z" name: success-200-isvc-70137.18c271edbe10aeb0 namespace: kserve-ci-e2e-test resourceVersion: "21619" uid: 98427b2f-7072-4b55-816b-ed1e9b065d63 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:08:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-70137 namespace: kserve-ci-e2e-test resourceVersion: "21612" uid: 37c02846-39e8-4700-a596-b0ef40e995d3 kind: Event lastTimestamp: "2026-07-15T11:08:37Z" message: InferenceService [success-200-isvc-70137] is Ready metadata: creationTimestamp: "2026-07-15T11:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:08:37Z" name: success-200-isvc-70137.18c271f91bda39ec namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 03f427eb-70b6-47bd-a01e-a6e751db132d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:17:03.243290Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw namespace: kserve-ci-e2e-test resourceVersion: "25377" uid: 58d5e7d5-59a8-4d71-a937-c32ac57ab1aa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bc774-predictor-5b6b775cdd-29slw to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw.18c2726ef323811a namespace: kserve-ci-e2e-test resourceVersion: "25382" uid: 19c3ea99-fd93-49e3-bfd0-d0497e1941d8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw namespace: kserve-ci-e2e-test resourceVersion: "25388" uid: 58d5e7d5-59a8-4d71-a937-c32ac57ab1aa kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw.18c2726f0d59e3e9 namespace: kserve-ci-e2e-test resourceVersion: "25426" uid: 5194bdcc-86e0-4c89-950c-6470534ee938 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw namespace: kserve-ci-e2e-test resourceVersion: "25380" uid: 58d5e7d5-59a8-4d71-a937-c32ac57ab1aa kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw.18c2726f0ebff257 namespace: kserve-ci-e2e-test resourceVersion: "25428" uid: 91b7852b-6ec6-4607-84be-780823bb2a37 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw namespace: kserve-ci-e2e-test resourceVersion: "25380" uid: 58d5e7d5-59a8-4d71-a937-c32ac57ab1aa kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw.18c2726f128798eb namespace: kserve-ci-e2e-test resourceVersion: "25430" uid: 46f0913d-a7e7-4e0a-ae67-b2cb375e0962 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw namespace: kserve-ci-e2e-test resourceVersion: "25380" uid: 58d5e7d5-59a8-4d71-a937-c32ac57ab1aa kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw.18c2726f1305f32d namespace: kserve-ci-e2e-test resourceVersion: "25431" uid: cf8d8456-f6d3-4841-b158-464eafe41160 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw namespace: kserve-ci-e2e-test resourceVersion: "25380" uid: 58d5e7d5-59a8-4d71-a937-c32ac57ab1aa kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw.18c2726f13119493 namespace: kserve-ci-e2e-test resourceVersion: "25432" uid: 32efb28e-23e1-43df-911c-3be6ecaeb252 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw namespace: kserve-ci-e2e-test resourceVersion: "25380" uid: 58d5e7d5-59a8-4d71-a937-c32ac57ab1aa kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw.18c2726f16e509b8 namespace: kserve-ci-e2e-test resourceVersion: "25436" uid: 8fd93015-9009-443d-b3c5-120016315d07 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw namespace: kserve-ci-e2e-test resourceVersion: "25380" uid: 58d5e7d5-59a8-4d71-a937-c32ac57ab1aa kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw.18c2726f1759c5a7 namespace: kserve-ci-e2e-test resourceVersion: "25437" uid: 05331739-3e81-4fe7-a060-ac5d6075c2bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:17:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw namespace: kserve-ci-e2e-test resourceVersion: "25380" uid: 58d5e7d5-59a8-4d71-a937-c32ac57ab1aa kind: Event lastTimestamp: "2026-07-15T11:17:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:17:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:41Z" name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw.18c2726f66dcce84 namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: a195800e-c736-42c7-870a-5c341ac6ad9c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw namespace: kserve-ci-e2e-test resourceVersion: "25380" uid: 58d5e7d5-59a8-4d71-a937-c32ac57ab1aa kind: Event lastTimestamp: "2026-07-15T11:34:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:34:22Z" name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw.18c27360eecb98af namespace: kserve-ci-e2e-test resourceVersion: "33367" uid: c37c2e70-0399-4195-bd12-3fffc6184b57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw namespace: kserve-ci-e2e-test resourceVersion: "25380" uid: 58d5e7d5-59a8-4d71-a937-c32ac57ab1aa kind: Event lastTimestamp: "2026-07-15T11:34:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:34:22Z" name: success-200-isvc-bc774-predictor-5b6b775cdd-29slw.18c27360eecc0c2c namespace: kserve-ci-e2e-test resourceVersion: "33368" uid: 974b75b8-f19e-408e-99d6-eddd64c39745 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bc774-predictor-5b6b775cdd namespace: kserve-ci-e2e-test resourceVersion: "25368" uid: c2fa66e6-d3cf-4e13-a9d9-2783c095e9ba kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: 'Created pod: success-200-isvc-bc774-predictor-5b6b775cdd-29slw' metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-bc774-predictor-5b6b775cdd.18c2726ef2b46917 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: 33bd2f2d-34a3-47dc-a3d0-cd872aa5f39a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bc774-predictor namespace: kserve-ci-e2e-test resourceVersion: "25365" uid: e2f5e1f6-fef8-47a7-8c34-9b579d935197 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: Scaled up replica set success-200-isvc-bc774-predictor-5b6b775cdd from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-bc774-predictor.18c2726ef0bcc7cb namespace: kserve-ci-e2e-test resourceVersion: "25369" uid: 787717f4-99de-4a7b-912f-da3dd574c94e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bc774-predictor namespace: kserve-ci-e2e-test resourceVersion: "25367" uid: 66493e4e-e60b-4f96-a0db-dd9b8c776d7c kind: Event lastTimestamp: "2026-07-15T11:17:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:18Z" name: success-200-isvc-bc774-predictor.18c272726fafef95 namespace: kserve-ci-e2e-test resourceVersion: "25593" uid: 1db061dc-4b9d-4711-81b4-6196ab9f0c90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bc774-predictor namespace: kserve-ci-e2e-test resourceVersion: "25367" uid: 66493e4e-e60b-4f96-a0db-dd9b8c776d7c kind: Event lastTimestamp: "2026-07-15T11:17:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:18Z" name: success-200-isvc-bc774-predictor.18c272726ffe14a2 namespace: kserve-ci-e2e-test resourceVersion: "25595" uid: 842a11c3-b5a4-40ad-b201-6df63be619c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bc774-predictor namespace: kserve-ci-e2e-test resourceVersion: "25594" uid: 66493e4e-e60b-4f96-a0db-dd9b8c776d7c kind: Event lastTimestamp: "2026-07-15T11:18:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:18Z" name: success-200-isvc-bc774-predictor.18c27275eec2075e namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: a0e90434-776e-4672-8991-d45a97f547c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bc774-predictor namespace: kserve-ci-e2e-test resourceVersion: "25594" uid: 66493e4e-e60b-4f96-a0db-dd9b8c776d7c kind: Event lastTimestamp: "2026-07-15T11:18:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:18Z" name: success-200-isvc-bc774-predictor.18c27275ef4f9235 namespace: kserve-ci-e2e-test resourceVersion: "26615" uid: e74d8765-8969-4aab-9fc7-7cd62e2d737c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bc774 namespace: kserve-ci-e2e-test resourceVersion: "25339" uid: 1ec6ea41-7ac6-4e77-ae2c-4dfb6cc629b1 kind: Event lastTimestamp: "2026-07-15T11:17:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:51Z" name: success-200-isvc-bc774.18c2726ef0c0843b namespace: kserve-ci-e2e-test resourceVersion: "26359" uid: c88d7b84-eeae-475e-b649-770e9c4c6ad8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bc774 namespace: kserve-ci-e2e-test resourceVersion: "25339" uid: 1ec6ea41-7ac6-4e77-ae2c-4dfb6cc629b1 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: 'Failed to update status for InferenceService "success-200-isvc-bc774": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bc774": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-bc774.18c2726ef11165fb namespace: kserve-ci-e2e-test resourceVersion: "25372" uid: 9d57fb83-fec8-49e6-9928-592341fb3716 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bc774 namespace: kserve-ci-e2e-test resourceVersion: "25339" uid: 1ec6ea41-7ac6-4e77-ae2c-4dfb6cc629b1 kind: Event lastTimestamp: "2026-07-15T11:17:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bc774": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:03Z" name: success-200-isvc-bc774.18c2726ef111f056 namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: 727f1d84-65b9-4b9b-86e0-e6aaa68c736c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bc774 namespace: kserve-ci-e2e-test resourceVersion: "26357" uid: 1ec6ea41-7ac6-4e77-ae2c-4dfb6cc629b1 kind: Event lastTimestamp: "2026-07-15T11:17:51Z" message: InferenceService [success-200-isvc-bc774] is Ready metadata: creationTimestamp: "2026-07-15T11:17:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:51Z" name: success-200-isvc-bc774.18c2727a2226933b namespace: kserve-ci-e2e-test resourceVersion: "26358" uid: 26d0de48-fc01-4d7f-aac6-ec9b861f7aff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:16:23.129734Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd namespace: kserve-ci-e2e-test resourceVersion: "24917" uid: 99dbd1d6-1f44-478c-a821-f786760dedd8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-cdb7a-predictor-54d956f49b-dzphd to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd.18c272659c2f305e namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: 48be2938-182d-4289-b988-8f8e72745375 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd namespace: kserve-ci-e2e-test resourceVersion: "24946" uid: 99dbd1d6-1f44-478c-a821-f786760dedd8 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd.18c27265b6bf469b namespace: kserve-ci-e2e-test resourceVersion: "24978" uid: 1c637ddc-c8b5-4d17-90b0-c6d17f5d7e5d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd namespace: kserve-ci-e2e-test resourceVersion: "24922" uid: 99dbd1d6-1f44-478c-a821-f786760dedd8 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd.18c27265b834b82f namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: 4486de99-9d5d-4233-90bc-38cb89c7abb8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd namespace: kserve-ci-e2e-test resourceVersion: "24922" uid: 99dbd1d6-1f44-478c-a821-f786760dedd8 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd.18c27265bc0c6fd4 namespace: kserve-ci-e2e-test resourceVersion: "24985" uid: 250df8c1-acf1-4ffb-b364-1acbb406d9e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd namespace: kserve-ci-e2e-test resourceVersion: "24922" uid: 99dbd1d6-1f44-478c-a821-f786760dedd8 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd.18c27265bc9a394c namespace: kserve-ci-e2e-test resourceVersion: "24986" uid: a41ca841-4fed-4063-bf1c-a1a05ab3e8ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd namespace: kserve-ci-e2e-test resourceVersion: "24922" uid: 99dbd1d6-1f44-478c-a821-f786760dedd8 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd.18c27265bca678e2 namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: fec0ef25-b5b5-4885-b161-27db9abf4b52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd namespace: kserve-ci-e2e-test resourceVersion: "24922" uid: 99dbd1d6-1f44-478c-a821-f786760dedd8 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd.18c27265bef486e6 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: 3b1fb094-e1d6-494b-9bc0-7d951f93e50b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd namespace: kserve-ci-e2e-test resourceVersion: "24922" uid: 99dbd1d6-1f44-478c-a821-f786760dedd8 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd.18c27265bf6b5363 namespace: kserve-ci-e2e-test resourceVersion: "24992" uid: 8f98a7a3-e9d7-4a27-9d64-bdbbe9b83aa7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd namespace: kserve-ci-e2e-test resourceVersion: "24922" uid: 99dbd1d6-1f44-478c-a821-f786760dedd8 kind: Event lastTimestamp: "2026-07-15T11:17:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:01Z" name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd.18c272660be708fc namespace: kserve-ci-e2e-test resourceVersion: "25291" uid: 908109e7-efb7-4719-9791-3e2c0d4a276c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd namespace: kserve-ci-e2e-test resourceVersion: "24922" uid: 99dbd1d6-1f44-478c-a821-f786760dedd8 kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:33Z" name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd.18c27275fb6af60d namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 2316c0ce-3eb9-40f2-9ccc-79855edeab36 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd namespace: kserve-ci-e2e-test resourceVersion: "24922" uid: 99dbd1d6-1f44-478c-a821-f786760dedd8 kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:33Z" name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd.18c27275fb6b5cff namespace: kserve-ci-e2e-test resourceVersion: "26122" uid: 11353911-a58e-41c7-a3f1-2d56b6dd7438 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd namespace: kserve-ci-e2e-test resourceVersion: "24922" uid: 99dbd1d6-1f44-478c-a821-f786760dedd8 kind: Event lastTimestamp: "2026-07-15T11:17:36Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:17:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:36Z" name: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd.18c272769465f71b namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: e711330a-43cd-4438-affc-33a495f70432 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-cdb7a-predictor-54d956f49b namespace: kserve-ci-e2e-test resourceVersion: "24903" uid: 7a363cea-6b06-47e4-bb13-dcf269bd0b01 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: 'Created pod: success-200-isvc-cdb7a-predictor-54d956f49b-dzphd' metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-cdb7a-predictor-54d956f49b.18c272659ba5c0bb namespace: kserve-ci-e2e-test resourceVersion: "24920" uid: 6812d687-9b7b-49d7-a7a1-9cbededbc91f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-cdb7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24899" uid: 839ed50d-c488-4642-88cd-12874b30ceb5 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Scaled up replica set success-200-isvc-cdb7a-predictor-54d956f49b from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-cdb7a-predictor.18c2726598027b40 namespace: kserve-ci-e2e-test resourceVersion: "24905" uid: 1ce285af-6421-4e7b-8d3a-b7a599dd80ef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:16:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cdb7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24902" uid: 297ac262-f7e3-488d-9ff6-3c8f9bbc5f8c kind: Event lastTimestamp: "2026-07-15T11:17:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:16:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:23Z" name: success-200-isvc-cdb7a-predictor.18c272691682ccc8 namespace: kserve-ci-e2e-test resourceVersion: "25636" uid: d1b115ea-6eca-4c64-9b81-38ed8f43dcbb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:16:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cdb7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24902" uid: 297ac262-f7e3-488d-9ff6-3c8f9bbc5f8c kind: Event lastTimestamp: "2026-07-15T11:17:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:16:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:23Z" name: success-200-isvc-cdb7a-predictor.18c2726916efb9be namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: e1f1b83e-109f-4d1a-8ab0-92853e2c4df7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cdb7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25119" uid: 297ac262-f7e3-488d-9ff6-3c8f9bbc5f8c kind: Event lastTimestamp: "2026-07-15T11:16:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:16:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:16:53Z" name: success-200-isvc-cdb7a-predictor.18c2726c95ad91dc namespace: kserve-ci-e2e-test resourceVersion: "25226" uid: bbaa422a-2f7a-44e9-93da-079d71915e74 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cdb7a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25119" uid: 297ac262-f7e3-488d-9ff6-3c8f9bbc5f8c kind: Event lastTimestamp: "2026-07-15T11:16:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:16:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:16:53Z" name: success-200-isvc-cdb7a-predictor.18c2726c961de4ec namespace: kserve-ci-e2e-test resourceVersion: "25229" uid: 6bc6229f-e3e1-49c9-a267-24b96fb2598b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cdb7a namespace: kserve-ci-e2e-test resourceVersion: "24881" uid: 817d4e2d-45bb-46d7-b66e-27c6d5296f34 kind: Event lastTimestamp: "2026-07-15T11:17:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:11Z" name: success-200-isvc-cdb7a.18c27265980caf64 namespace: kserve-ci-e2e-test resourceVersion: "25542" uid: 7ef1c141-485d-49f9-b3d5-ec93a734ebf6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cdb7a namespace: kserve-ci-e2e-test resourceVersion: "24881" uid: 817d4e2d-45bb-46d7-b66e-27c6d5296f34 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: 'Failed to update status for InferenceService "success-200-isvc-cdb7a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cdb7a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-cdb7a.18c27265987db09c namespace: kserve-ci-e2e-test resourceVersion: "24910" uid: 4542a48e-98cb-45d9-bec8-b2e6f0444f50 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cdb7a namespace: kserve-ci-e2e-test resourceVersion: "24881" uid: 817d4e2d-45bb-46d7-b66e-27c6d5296f34 kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cdb7a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-cdb7a.18c27265987e1bf8 namespace: kserve-ci-e2e-test resourceVersion: "24911" uid: 183fa720-59b8-4bdb-8965-337bbfbd4529 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cdb7a namespace: kserve-ci-e2e-test resourceVersion: "25535" uid: 817d4e2d-45bb-46d7-b66e-27c6d5296f34 kind: Event lastTimestamp: "2026-07-15T11:17:11Z" message: InferenceService [success-200-isvc-cdb7a] is Ready metadata: creationTimestamp: "2026-07-15T11:17:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:11Z" name: success-200-isvc-cdb7a.18c27270ca6b777e namespace: kserve-ci-e2e-test resourceVersion: "25536" uid: 38e0252b-a166-41ea-81a0-fce9eb2f8c23 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:05:47.986769Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 4e971415-7cf0-42c0-88b8-0caac82999b6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-cf436-predictor-69cbcd7974-z8klj to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:05:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:05:47Z" name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj.18c271d1bab6b338 namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: 77aa311e-8ae3-4177-ac46-f5bbc5e9638a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj namespace: kserve-ci-e2e-test resourceVersion: "19614" uid: 4e971415-7cf0-42c0-88b8-0caac82999b6 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:05:48Z" name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj.18c271d1d3f3e6c0 namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 5c0a01c2-4b10-47ff-81ce-25806e837b53 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 4e971415-7cf0-42c0-88b8-0caac82999b6 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj.18c271d1d5c4f2b4 namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: 029e1537-71dd-4a5d-a18e-5b4f1285bd0b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 4e971415-7cf0-42c0-88b8-0caac82999b6 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj.18c271d1d9650204 namespace: kserve-ci-e2e-test resourceVersion: "19643" uid: 3acf6695-7def-468c-af6c-94d55a665123 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 4e971415-7cf0-42c0-88b8-0caac82999b6 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj.18c271d1d9d22f66 namespace: kserve-ci-e2e-test resourceVersion: "19645" uid: 5f3d4e96-0be8-4657-b6ff-76b59669c201 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 4e971415-7cf0-42c0-88b8-0caac82999b6 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj.18c271d1d9dbb242 namespace: kserve-ci-e2e-test resourceVersion: "19646" uid: 28de99c7-a2a0-4d47-8f34-420eddb928ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 4e971415-7cf0-42c0-88b8-0caac82999b6 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj.18c271d1dcc292fc namespace: kserve-ci-e2e-test resourceVersion: "19649" uid: 3e11f035-e265-46e3-915b-ba8a0411eb6a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 4e971415-7cf0-42c0-88b8-0caac82999b6 kind: Event lastTimestamp: "2026-07-15T11:05:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:48Z" name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj.18c271d1dd371af2 namespace: kserve-ci-e2e-test resourceVersion: "19651" uid: 29d4c0af-7b02-4f96-9651-3d12498735c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-15T11:05:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 4e971415-7cf0-42c0-88b8-0caac82999b6 kind: Event lastTimestamp: "2026-07-15T11:06:35Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:35Z" name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj.18c271d213ac4f64 namespace: kserve-ci-e2e-test resourceVersion: "20171" uid: 68bd1b45-1113-46a0-a4d5-7597e25e3789 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 4e971415-7cf0-42c0-88b8-0caac82999b6 kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj.18c271e46b988b2b namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: 4f123077-b1d0-4757-a272-8d0d91322181 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 4e971415-7cf0-42c0-88b8-0caac82999b6 kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj.18c271e46b994b35 namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 547bef98-5bd2-433b-b707-b43ca1aab386 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 4e971415-7cf0-42c0-88b8-0caac82999b6 kind: Event lastTimestamp: "2026-07-15T11:07:10Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:10Z" name: success-200-isvc-cf436-predictor-69cbcd7974-z8klj.18c271e4effc0b58 namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: b0655a7f-8385-4229-9a87-ca58491822f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-cf436-predictor-69cbcd7974 namespace: kserve-ci-e2e-test resourceVersion: "19568" uid: 4fa0af8e-4a35-4e62-9ebd-49d1f5cb2b60 kind: Event lastTimestamp: "2026-07-15T11:05:47Z" message: 'Created pod: success-200-isvc-cf436-predictor-69cbcd7974-z8klj' metadata: creationTimestamp: "2026-07-15T11:05:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:05:47Z" name: success-200-isvc-cf436-predictor-69cbcd7974.18c271d1b9e3010c namespace: kserve-ci-e2e-test resourceVersion: "19592" uid: 6a5e8f2b-c52d-40dd-bb36-639151598722 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-cf436-predictor namespace: kserve-ci-e2e-test resourceVersion: "19564" uid: 463dc82a-bef3-4d28-b8bf-2f2f4d2bf550 kind: Event lastTimestamp: "2026-07-15T11:05:47Z" message: Scaled up replica set success-200-isvc-cf436-predictor-69cbcd7974 from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:05:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:05:47Z" name: success-200-isvc-cf436-predictor.18c271d1b63724ac namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: 2058329d-6496-4f39-beae-92f8bdb64ff8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cf436-predictor namespace: kserve-ci-e2e-test resourceVersion: "19566" uid: f796f2d2-7001-4e00-a9ca-c46ed990aa89 kind: Event lastTimestamp: "2026-07-15T11:06:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:02Z" name: success-200-isvc-cf436-predictor.18c271d534e7586e namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: c163afa3-661f-4591-b0d9-1207ab380e1d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cf436-predictor namespace: kserve-ci-e2e-test resourceVersion: "19566" uid: f796f2d2-7001-4e00-a9ca-c46ed990aa89 kind: Event lastTimestamp: "2026-07-15T11:06:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:06:02Z" name: success-200-isvc-cf436-predictor.18c271d53564173a namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 0f6fff5c-14d7-4957-9a8b-764f98274658 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-15T11:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cf436-predictor namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: f796f2d2-7001-4e00-a9ca-c46ed990aa89 kind: Event lastTimestamp: "2026-07-15T11:07:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:03Z" name: success-200-isvc-cf436-predictor.18c271d8b4425bbd namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: 84987b84-e40f-4bae-aabc-f9fcc7c4bdea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-15T11:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cf436-predictor namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: f796f2d2-7001-4e00-a9ca-c46ed990aa89 kind: Event lastTimestamp: "2026-07-15T11:07:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:03Z" name: success-200-isvc-cf436-predictor.18c271d8b4d089f1 namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 69fcf19f-7466-485a-83bc-a015ab713e53 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:05:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cf436 namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: 56d8e682-05ad-4082-b97f-2ad99da94bc9 kind: Event lastTimestamp: "2026-07-15T11:06:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:05:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:06:45Z" name: success-200-isvc-cf436.18c271d1b638f424 namespace: kserve-ci-e2e-test resourceVersion: "20274" uid: 74f4286e-5b56-4b3b-8a12-ecc93a63c02d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cf436 namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: 56d8e682-05ad-4082-b97f-2ad99da94bc9 kind: Event lastTimestamp: "2026-07-15T11:05:47Z" message: 'Failed to update status for InferenceService "success-200-isvc-cf436": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cf436": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:05:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:05:47Z" name: success-200-isvc-cf436.18c271d1b6b4c491 namespace: kserve-ci-e2e-test resourceVersion: "19573" uid: c1bee5c3-767d-49a1-97b3-933ae4e70f31 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cf436 namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: 56d8e682-05ad-4082-b97f-2ad99da94bc9 kind: Event lastTimestamp: "2026-07-15T11:05:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cf436": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:05:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:05:47Z" name: success-200-isvc-cf436.18c271d1b6b52eb2 namespace: kserve-ci-e2e-test resourceVersion: "19574" uid: a29fc2c9-1727-468e-a0d4-704dd63bfe60 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:06:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cf436 namespace: kserve-ci-e2e-test resourceVersion: "20272" uid: 56d8e682-05ad-4082-b97f-2ad99da94bc9 kind: Event lastTimestamp: "2026-07-15T11:06:45Z" message: InferenceService [success-200-isvc-cf436] is Ready metadata: creationTimestamp: "2026-07-15T11:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:06:45Z" name: success-200-isvc-cf436.18c271df23395721 namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: 7aca8546-5057-4230-a346-1974a765e43c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:17:33.477957Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf namespace: kserve-ci-e2e-test resourceVersion: "26123" uid: 97287f11-af35-42b1-9806-315ef67e4740 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:17:33Z" name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf.18c27275fd43e41f namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: c35f12e8-3938-4264-9094-4361f7e0f654 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 97287f11-af35-42b1-9806-315ef67e4740 kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:17:33Z" name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf.18c2727617375164 namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: 4334a7be-ba83-4a2b-9e51-20d9a4cceabb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 97287f11-af35-42b1-9806-315ef67e4740 kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:33Z" name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf.18c27276188f62ce namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: 54022a40-e7b9-4e31-8fdf-52c26d00ca77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 97287f11-af35-42b1-9806-315ef67e4740 kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:33Z" name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf.18c272761ba81626 namespace: kserve-ci-e2e-test resourceVersion: "26181" uid: 36672c29-ced5-4176-89f6-a5d1f92cc3ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 97287f11-af35-42b1-9806-315ef67e4740 kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:33Z" name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf.18c272761c1ab0df namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: ebbef53f-b3db-4ebf-9320-b6a39edf8397 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 97287f11-af35-42b1-9806-315ef67e4740 kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:34Z" name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf.18c272761c24e91d namespace: kserve-ci-e2e-test resourceVersion: "26183" uid: 01634308-db8b-426c-a53b-69731e970981 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 97287f11-af35-42b1-9806-315ef67e4740 kind: Event lastTimestamp: "2026-07-15T11:17:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:34Z" name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf.18c272761ecbd905 namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: 39f8f3fe-4a28-4365-8b0a-45494c3ac5bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 97287f11-af35-42b1-9806-315ef67e4740 kind: Event lastTimestamp: "2026-07-15T11:17:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:17:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:17:34Z" name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf.18c272761f3f107e namespace: kserve-ci-e2e-test resourceVersion: "26185" uid: 6ebc7fed-4cc5-4d3b-a795-22b80edd6c24 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-15T11:17:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 97287f11-af35-42b1-9806-315ef67e4740 kind: Event lastTimestamp: "2026-07-15T11:26:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:17:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:26:51Z" name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf.18c272766ba537dd namespace: kserve-ci-e2e-test resourceVersion: "30263" uid: 2f681c14-f30e-4c40-a308-9892f5365439 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:26:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 97287f11-af35-42b1-9806-315ef67e4740 kind: Event lastTimestamp: "2026-07-15T11:26:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:26:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:26:48Z" name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf.18c272f7376ec4ef namespace: kserve-ci-e2e-test resourceVersion: "30222" uid: ea63623a-21e2-4f12-ac1c-f6e032a662ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:26:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 97287f11-af35-42b1-9806-315ef67e4740 kind: Event lastTimestamp: "2026-07-15T11:26:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:26:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:26:48Z" name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf.18c272f7376fe70f namespace: kserve-ci-e2e-test resourceVersion: "30223" uid: b63b1ced-49df-4a9d-82a4-0dad681cc3b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 97287f11-af35-42b1-9806-315ef67e4740 kind: Event lastTimestamp: "2026-07-15T11:26:51Z" message: 'Readiness probe failed: Get "https://10.133.0.42:8643/healthz": dial tcp 10.133.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:26:51Z" name: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf.18c272f7e03e8d95 namespace: kserve-ci-e2e-test resourceVersion: "30260" uid: 88540dcf-ba1a-4719-a363-3a76e945ffee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d1ffd-predictor-759c7cbb88 namespace: kserve-ci-e2e-test resourceVersion: "26111" uid: fd381a01-705b-4d11-a1af-13731b7f8873 kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: 'Created pod: success-200-isvc-d1ffd-predictor-759c7cbb88-dbdcf' metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:33Z" name: success-200-isvc-d1ffd-predictor-759c7cbb88.18c27275fcb59755 namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: eb7fd4d3-515b-4808-b598-75c8cd9a06b8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d1ffd-predictor namespace: kserve-ci-e2e-test resourceVersion: "26105" uid: 0d738747-e27e-4c7f-b4e2-eee8bed30ac1 kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: Scaled up replica set success-200-isvc-d1ffd-predictor-759c7cbb88 from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:17:33Z" name: success-200-isvc-d1ffd-predictor.18c27275fa37c0d3 namespace: kserve-ci-e2e-test resourceVersion: "26113" uid: 41619e92-b2ec-48fc-add3-135f82cad7fb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:17:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1ffd-predictor namespace: kserve-ci-e2e-test resourceVersion: "26109" uid: afbdd8bd-1667-4900-999c-b1106b69c445 kind: Event lastTimestamp: "2026-07-15T11:18:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:03Z" name: success-200-isvc-d1ffd-predictor.18c27279789da161 namespace: kserve-ci-e2e-test resourceVersion: "26495" uid: 220c257f-7cb0-463b-ba92-ee6e571def49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:17:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1ffd-predictor namespace: kserve-ci-e2e-test resourceVersion: "26109" uid: afbdd8bd-1667-4900-999c-b1106b69c445 kind: Event lastTimestamp: "2026-07-15T11:18:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:03Z" name: success-200-isvc-d1ffd-predictor.18c2727978f5045b namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: c65fcbb5-9da7-4ea6-b3b3-56bcdb0ced16 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:18:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1ffd-predictor namespace: kserve-ci-e2e-test resourceVersion: "26325" uid: afbdd8bd-1667-4900-999c-b1106b69c445 kind: Event lastTimestamp: "2026-07-15T11:18:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:48Z" name: success-200-isvc-d1ffd-predictor.18c2728076712432 namespace: kserve-ci-e2e-test resourceVersion: "26901" uid: 859192b6-f555-4039-8a56-9ea2863f8787 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T11:18:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1ffd-predictor namespace: kserve-ci-e2e-test resourceVersion: "26325" uid: afbdd8bd-1667-4900-999c-b1106b69c445 kind: Event lastTimestamp: "2026-07-15T11:18:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:48Z" name: success-200-isvc-d1ffd-predictor.18c2728076e5f50e namespace: kserve-ci-e2e-test resourceVersion: "26902" uid: 0fd45771-0f58-44c1-be0c-a89987e49f62 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1ffd namespace: kserve-ci-e2e-test resourceVersion: "26081" uid: 30fbf314-7304-4973-8947-0fb095653a61 kind: Event lastTimestamp: "2026-07-15T11:18:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:18:21Z" name: success-200-isvc-d1ffd.18c27275fa3038ad namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: abd522c6-a1eb-48a4-8812-09837417ce00 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1ffd namespace: kserve-ci-e2e-test resourceVersion: "26081" uid: 30fbf314-7304-4973-8947-0fb095653a61 kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: 'Failed to update status for InferenceService "success-200-isvc-d1ffd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d1ffd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:33Z" name: success-200-isvc-d1ffd.18c27275faf0f83b namespace: kserve-ci-e2e-test resourceVersion: "26118" uid: 8a7e516f-1d4d-4d05-acd9-0d363e9d43a5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:17:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1ffd namespace: kserve-ci-e2e-test resourceVersion: "26081" uid: 30fbf314-7304-4973-8947-0fb095653a61 kind: Event lastTimestamp: "2026-07-15T11:17:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d1ffd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:17:33Z" name: success-200-isvc-d1ffd.18c27275faf1711d namespace: kserve-ci-e2e-test resourceVersion: "26119" uid: 7dcaecc3-83cf-47b2-8c92-7778ffa94b6e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1ffd namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: 30fbf314-7304-4973-8947-0fb095653a61 kind: Event lastTimestamp: "2026-07-15T11:18:21Z" message: InferenceService [success-200-isvc-d1ffd] is Ready metadata: creationTimestamp: "2026-07-15T11:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:18:21Z" name: success-200-isvc-d1ffd.18c27281278f6c37 namespace: kserve-ci-e2e-test resourceVersion: "26664" uid: 5f8e9cfe-9eff-42a3-b09f-3c87b71a9aa6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:07:08.361597Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 84640095-4359-4999-b3c9-7d7602b8daca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d6666-predictor-8674b65f5d-8hxmp to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp.18c271e4716d3220 namespace: kserve-ci-e2e-test resourceVersion: "20571" uid: 44892e06-c5ff-475a-a764-807287cd169d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: 84640095-4359-4999-b3c9-7d7602b8daca kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp.18c271e48b39102e namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 6a68c9c6-91d5-4533-ab9f-f4f67524427f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 84640095-4359-4999-b3c9-7d7602b8daca kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp.18c271e48ca0c5e5 namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: e975986e-f20d-4603-b147-5c0dfd7e8ef5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 84640095-4359-4999-b3c9-7d7602b8daca kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp.18c271e48f587fef namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 1de0ad90-15ce-4e89-a119-67c070e9418d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 84640095-4359-4999-b3c9-7d7602b8daca kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp.18c271e48fcd3d36 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: e0799ba5-b4bf-4ed4-9ec9-197fd1bd1bf1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 84640095-4359-4999-b3c9-7d7602b8daca kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp.18c271e48fd6c320 namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 40a8d98c-e1d2-40b6-8375-3bdfe81d5402 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 84640095-4359-4999-b3c9-7d7602b8daca kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp.18c271e4927fe4f5 namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: d0b70878-5234-4755-98c9-5d7991991b55 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 84640095-4359-4999-b3c9-7d7602b8daca kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp.18c271e492ee3e84 namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: d816bf20-b6a3-42e5-80b5-76462b28cc68 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 84640095-4359-4999-b3c9-7d7602b8daca kind: Event lastTimestamp: "2026-07-15T11:07:46Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T11:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:07:46Z" name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp.18c271e50542e27c namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: 83c88693-f430-452f-9ae6-32d355aaeef7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 84640095-4359-4999-b3c9-7d7602b8daca kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp.18c2726597c22ba5 namespace: kserve-ci-e2e-test resourceVersion: "24904" uid: 63265523-97bd-4d6e-b453-403c58405561 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:16:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 84640095-4359-4999-b3c9-7d7602b8daca kind: Event lastTimestamp: "2026-07-15T11:16:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:16:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:16:23Z" name: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp.18c2726597c2b6d6 namespace: kserve-ci-e2e-test resourceVersion: "24907" uid: 277b4b8e-0ed9-46c8-98b7-edc616677a63 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d6666-predictor-8674b65f5d namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: c6251f7a-1908-4d44-a5a1-f804fec2533e kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: 'Created pod: success-200-isvc-d6666-predictor-8674b65f5d-8hxmp' metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-d6666-predictor-8674b65f5d.18c271e470e8a057 namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: 40eb69d5-14e6-4c22-8fdb-95d941221279 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d6666-predictor namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: cbfe2b47-aebd-4b86-a37e-4127f5e69952 kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: Scaled up replica set success-200-isvc-d6666-predictor-8674b65f5d from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-d6666-predictor.18c271e46e9e3974 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 4c1231aa-6f8f-43b9-8dec-6f94160c6ae1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:07:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d6666-predictor namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: ae6c44f5-c453-42c3-a54a-7a087e66b165 kind: Event lastTimestamp: "2026-07-15T11:08:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:07:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:38Z" name: success-200-isvc-d6666-predictor.18c271e7ece2b024 namespace: kserve-ci-e2e-test resourceVersion: "21628" uid: 713e81b1-da3c-4a49-a037-a762a2884180 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:07:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d6666-predictor namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: ae6c44f5-c453-42c3-a54a-7a087e66b165 kind: Event lastTimestamp: "2026-07-15T11:08:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T11:07:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:08:38Z" name: success-200-isvc-d6666-predictor.18c271e7ed5787f4 namespace: kserve-ci-e2e-test resourceVersion: "21630" uid: 86bc0a05-2fdb-407a-81e6-06c35aef4404 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d6666 namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 56c947e0-d195-47d6-a09c-4651d1c3d710 kind: Event lastTimestamp: "2026-07-15T11:07:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:07:56Z" name: success-200-isvc-d6666.18c271e46eac921f namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: 75cc68f6-796b-47bc-91dd-8afc29d9a564 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d6666 namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 56c947e0-d195-47d6-a09c-4651d1c3d710 kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: 'Failed to update status for InferenceService "success-200-isvc-d6666": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d6666": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-d6666.18c271e46f6db980 namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: c3d14e6d-33e9-47a3-abc7-912ccff92d39 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d6666 namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 56c947e0-d195-47d6-a09c-4651d1c3d710 kind: Event lastTimestamp: "2026-07-15T11:07:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d6666": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T11:07:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:07:08Z" name: success-200-isvc-d6666.18c271e46f6dfee0 namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: 19e7ce80-f1b1-4f30-b642-d3b8d36887ce reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:07:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d6666 namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 56c947e0-d195-47d6-a09c-4651d1c3d710 kind: Event lastTimestamp: "2026-07-15T11:07:56Z" message: InferenceService [success-200-isvc-d6666] is Ready metadata: creationTimestamp: "2026-07-15T11:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:07:56Z" name: success-200-isvc-d6666.18c271efc1712620 namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 436495fa-bd93-4bf4-b54d-7f95e7e85514 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T10:56:33.193931Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: c456ffb4-785e-468a-9715-dd367745b6af kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9 to ip-10-0-131-34.ec2.internal metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9.18c271508e7bd014 namespace: kserve-ci-e2e-test resourceVersion: "15192" uid: ee013ec3-4eca-4d07-af19-8e06b53219f3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9 namespace: kserve-ci-e2e-test resourceVersion: "15200" uid: c456ffb4-785e-468a-9715-dd367745b6af kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9.18c27150a7ca5f3a namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: f9263c0f-c49e-4d7b-bb0d-65b95b05a5ad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: c456ffb4-785e-468a-9715-dd367745b6af kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1738" already present on machine metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9.18c27150a95d8ba9 namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: c78b238e-daaa-4a7b-aff2-3ebd7db7d7d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: c456ffb4-785e-468a-9715-dd367745b6af kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9.18c27150acbb2495 namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 52302fc0-a138-4035-b5e2-c1a7d9679d57 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: c456ffb4-785e-468a-9715-dd367745b6af kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9.18c27150ad3ec9fa namespace: kserve-ci-e2e-test resourceVersion: "15259" uid: 5a14589e-6a27-4dc3-8079-8edcfa34dd58 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: c456ffb4-785e-468a-9715-dd367745b6af kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9.18c27150ad4a7242 namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: 5135211f-ad8e-4009-8e97-b01bacffd380 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: c456ffb4-785e-468a-9715-dd367745b6af kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9.18c27150b028f52c namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: 0fd3fdfa-f831-44d6-88eb-2ac269498060 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: c456ffb4-785e-468a-9715-dd367745b6af kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9.18c27150b0a85377 namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 5a20166a-8fff-457b-a7bd-7809458f3b45 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T10:56:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: c456ffb4-785e-468a-9715-dd367745b6af kind: Event lastTimestamp: "2026-07-15T10:57:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-15T10:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:11Z" name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9.18c271510b8d0a19 namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 6c0977b2-9f67-477d-b38b-7adc2669c175 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: c456ffb4-785e-468a-9715-dd367745b6af kind: Event lastTimestamp: "2026-07-15T11:05:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-15T11:05:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:47Z" name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9.18c271d1b54759d5 namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: 87137467-5718-4659-a8ff-7054c8e18619 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: c456ffb4-785e-468a-9715-dd367745b6af kind: Event lastTimestamp: "2026-07-15T11:05:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-15T11:05:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:47Z" name: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9.18c271d1b547f873 namespace: kserve-ci-e2e-test resourceVersion: "19567" uid: 157cf70b-d5b2-412b-961d-624a86ee0292 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-34.ec2.internal source: component: kubelet host: ip-10-0-131-34.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ff4f8-predictor-67c88c6d66 namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: f8918d60-ca74-4c36-8e07-45a70d833704 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'Created pod: success-200-isvc-ff4f8-predictor-67c88c6d66-jxbd9' metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-ff4f8-predictor-67c88c6d66.18c271508d4ff19f namespace: kserve-ci-e2e-test resourceVersion: "15189" uid: a6dc53f5-5977-4968-9939-776f5b6e5a07 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ff4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: a20a0f81-df9c-42df-afdf-62608a3c8631 kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: Scaled up replica set success-200-isvc-ff4f8-predictor-67c88c6d66 from 0 to 1 metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-ff4f8-predictor.18c27150895b499d namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: 7db0b875-e3ba-4cde-8fa0-457f5de4fd1f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ff4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: dbff89b0-0b65-4673-9b6b-b5f0b4d9cda8 kind: Event lastTimestamp: "2026-07-15T10:56:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:48Z" name: success-200-isvc-ff4f8-predictor.18c2715407edfb43 namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: ca56d924-fc9a-462a-9bbd-f00e93b2bcfc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ff4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: dbff89b0-0b65-4673-9b6b-b5f0b4d9cda8 kind: Event lastTimestamp: "2026-07-15T10:56:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:48Z" name: success-200-isvc-ff4f8-predictor.18c2715408bb96cc namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: a8363189-890f-484c-9872-03ac9b32c1d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:57:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ff4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: dbff89b0-0b65-4673-9b6b-b5f0b4d9cda8 kind: Event lastTimestamp: "2026-07-15T10:57:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:33Z" name: success-200-isvc-ff4f8-predictor.18c27157879e2a57 namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: 295c8792-4d0e-4833-b2de-66023357fe04 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-15T10:57:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ff4f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: dbff89b0-0b65-4673-9b6b-b5f0b4d9cda8 kind: Event lastTimestamp: "2026-07-15T10:57:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-15T10:57:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:33Z" name: success-200-isvc-ff4f8-predictor.18c27157881036d8 namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 63942555-a225-4046-accc-31f54cd962bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ff4f8 namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 5150d531-e863-42af-ae11-078e9b37cf1f kind: Event lastTimestamp: "2026-07-15T10:57:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:57:21Z" name: success-200-isvc-ff4f8.18c271508980613e namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 6ed95607-71c5-49c8-a2a8-40972515bc18 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ff4f8 namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 5150d531-e863-42af-ae11-078e9b37cf1f kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'Failed to update status for InferenceService "success-200-isvc-ff4f8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ff4f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-ff4f8.18c271508a1c8558 namespace: kserve-ci-e2e-test resourceVersion: "15179" uid: 46949b96-2b46-42c6-8375-addae6af607b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ff4f8 namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 5150d531-e863-42af-ae11-078e9b37cf1f kind: Event lastTimestamp: "2026-07-15T10:56:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ff4f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-15T10:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:56:33Z" name: success-200-isvc-ff4f8.18c271508a1ccbd1 namespace: kserve-ci-e2e-test resourceVersion: "15181" uid: e9351575-4b29-4f87-aa94-253cbeda9e49 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ff4f8 namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 5150d531-e863-42af-ae11-078e9b37cf1f kind: Event lastTimestamp: "2026-07-15T10:57:21Z" message: InferenceService [success-200-isvc-ff4f8] is Ready metadata: creationTimestamp: "2026-07-15T10:57:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:57:21Z" name: success-200-isvc-ff4f8.18c2715bc73e1424 namespace: kserve-ci-e2e-test resourceVersion: "15908" uid: 0eb9fd36-8030-4c08-8936-57d25626e6b6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T10:56:22.761221Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6db9e-5cb89b98dc-7x78t namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: 3760677f-62e7-4d20-bb5a-a8dca199de7d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-6db9e-5cb89b98dc-7x78t to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-07-15T10:56:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T10:56:22Z" name: switch-graph-6db9e-5cb89b98dc-7x78t.18c2714e20a5403d namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: 8280d5a2-c2fd-43ce-b1d8-ee23339ec5c5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:22Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6db9e-5cb89b98dc-7x78t namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 3760677f-62e7-4d20-bb5a-a8dca199de7d kind: Event lastTimestamp: "2026-07-15T10:56:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-6db9e-serving-cert" not found' metadata: creationTimestamp: "2026-07-15T10:56:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:22Z" name: switch-graph-6db9e-5cb89b98dc-7x78t.18c2714e28776df4 namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: 05b85684-6882-451e-a6da-69dd84f7c0a2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:23Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6db9e-5cb89b98dc-7x78t namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: 3760677f-62e7-4d20-bb5a-a8dca199de7d kind: Event lastTimestamp: "2026-07-15T10:56:23Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T10:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T10:56:23Z" name: switch-graph-6db9e-5cb89b98dc-7x78t.18c2714e5e5a2e98 namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a533aafe-2a7f-4353-a89b-998ce4ae1b88 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6db9e} kind: Pod name: switch-graph-6db9e-5cb89b98dc-7x78t namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 3760677f-62e7-4d20-bb5a-a8dca199de7d kind: Event lastTimestamp: "2026-07-15T10:56:23Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:0e1a169e211c4cdcd19b5d9ae2b04ed590c1bcb2f57424f5c6e7d0d0f2faa1f0" metadata: creationTimestamp: "2026-07-15T10:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:23Z" name: switch-graph-6db9e-5cb89b98dc-7x78t.18c2714e5fb2aeb6 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 726e33e8-48d0-4721-8c27-1ca06b29aaf5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6db9e} kind: Pod name: switch-graph-6db9e-5cb89b98dc-7x78t namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 3760677f-62e7-4d20-bb5a-a8dca199de7d kind: Event lastTimestamp: "2026-07-15T10:56:26Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:0e1a169e211c4cdcd19b5d9ae2b04ed590c1bcb2f57424f5c6e7d0d0f2faa1f0" in 2.238s (2.238s including waiting). Image size: 218077281 bytes.' metadata: creationTimestamp: "2026-07-15T10:56:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:26Z" name: switch-graph-6db9e-5cb89b98dc-7x78t.18c2714ee5269ed9 namespace: kserve-ci-e2e-test resourceVersion: "15061" uid: c4fd0b9e-8944-4e94-a578-beb702f26641 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6db9e} kind: Pod name: switch-graph-6db9e-5cb89b98dc-7x78t namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 3760677f-62e7-4d20-bb5a-a8dca199de7d kind: Event lastTimestamp: "2026-07-15T10:56:26Z" message: 'Created container: switch-graph-6db9e' metadata: creationTimestamp: "2026-07-15T10:56:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:26Z" name: switch-graph-6db9e-5cb89b98dc-7x78t.18c2714ee812cf84 namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: 3a0efdce-5eed-4050-adb6-964ef392d5b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6db9e} kind: Pod name: switch-graph-6db9e-5cb89b98dc-7x78t namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 3760677f-62e7-4d20-bb5a-a8dca199de7d kind: Event lastTimestamp: "2026-07-15T10:56:26Z" message: Started container switch-graph-6db9e metadata: creationTimestamp: "2026-07-15T10:56:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:26Z" name: switch-graph-6db9e-5cb89b98dc-7x78t.18c2714ee88780e5 namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: 17d8dad0-fe4f-434b-9b51-541721cfd5fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6db9e} kind: Pod name: switch-graph-6db9e-5cb89b98dc-7x78t namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 3760677f-62e7-4d20-bb5a-a8dca199de7d kind: Event lastTimestamp: "2026-07-15T10:56:32Z" message: Stopping container switch-graph-6db9e metadata: creationTimestamp: "2026-07-15T10:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:56:32Z" name: switch-graph-6db9e-5cb89b98dc-7x78t.18c271507fe5c986 namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 6967bbee-9ede-426c-88ce-a119b387ae82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T10:56:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6db9e} kind: Pod name: switch-graph-6db9e-5cb89b98dc-7x78t namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 3760677f-62e7-4d20-bb5a-a8dca199de7d kind: Event lastTimestamp: "2026-07-15T10:57:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-15T10:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:02Z" name: switch-graph-6db9e-5cb89b98dc-7x78t.18c2715196d0547b namespace: kserve-ci-e2e-test resourceVersion: "15528" uid: acbc027b-8f0b-4481-ba7a-c603b64ba2f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-6db9e-5cb89b98dc namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: 8805c192-e79d-4a83-9323-b17f833cad69 kind: Event lastTimestamp: "2026-07-15T10:56:22Z" message: 'Created pod: switch-graph-6db9e-5cb89b98dc-7x78t' metadata: creationTimestamp: "2026-07-15T10:56:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:22Z" name: switch-graph-6db9e-5cb89b98dc.18c2714e2023123a namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: 00dea244-4f3a-44fb-a39e-02dac790d952 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-6db9e namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: c82fdcd7-afd2-46c5-9f88-d084a5a35daf kind: Event lastTimestamp: "2026-07-15T10:56:22Z" message: Scaled up replica set switch-graph-6db9e-5cb89b98dc from 0 to 1 metadata: creationTimestamp: "2026-07-15T10:56:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:56:22Z" name: switch-graph-6db9e.18c2714e1f187b97 namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: a78685f7-b0c7-4d1a-bf75-2b55f9920eb9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:56:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-6db9e namespace: kserve-ci-e2e-test resourceVersion: "15027" uid: 7983c9bc-080f-4d10-9558-ecf1182616b0 kind: Event lastTimestamp: "2026-07-15T10:56:22Z" message: InferenceGraph [switch-graph-6db9e] is Ready metadata: creationTimestamp: "2026-07-15T10:56:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:56:22Z" name: switch-graph-6db9e.18c2714e249b47b0 namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: 645cc19a-2c4a-49c3-8bd4-77ad8ac2b0fa reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-15T11:18:03.312282Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-bc774-7778877fb9-g5794 namespace: kserve-ci-e2e-test resourceVersion: "26473" uid: 8fc0b3a5-fbf0-4cac-b4f8-be3a4d729942 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-bc774-7778877fb9-g5794 to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-07-15T11:18:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T11:18:03Z" name: switch-graph-bc774-7778877fb9-g5794.18c2727cef8791a3 namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: 00153af0-dc1f-4021-af3d-fad432f49814 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:03Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-bc774-7778877fb9-g5794 namespace: kserve-ci-e2e-test resourceVersion: "26482" uid: 8fc0b3a5-fbf0-4cac-b4f8-be3a4d729942 kind: Event lastTimestamp: "2026-07-15T11:18:03Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T11:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T11:18:03Z" name: switch-graph-bc774-7778877fb9-g5794.18c2727d0a37472b namespace: kserve-ci-e2e-test resourceVersion: "26505" uid: f44cd10f-08ae-43a0-8589-0b6b543b3259 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bc774} kind: Pod name: switch-graph-bc774-7778877fb9-g5794 namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 8fc0b3a5-fbf0-4cac-b4f8-be3a4d729942 kind: Event lastTimestamp: "2026-07-15T11:18:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0e1a169e211c4cdcd19b5d9ae2b04ed590c1bcb2f57424f5c6e7d0d0f2faa1f0" already present on machine metadata: creationTimestamp: "2026-07-15T11:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:18:03Z" name: switch-graph-bc774-7778877fb9-g5794.18c2727d0b929d52 namespace: kserve-ci-e2e-test resourceVersion: "26507" uid: e951f865-d5d7-43f4-a989-bc3646b3aa14 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bc774} kind: Pod name: switch-graph-bc774-7778877fb9-g5794 namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 8fc0b3a5-fbf0-4cac-b4f8-be3a4d729942 kind: Event lastTimestamp: "2026-07-15T11:18:03Z" message: 'Created container: switch-graph-bc774' metadata: creationTimestamp: "2026-07-15T11:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:18:03Z" name: switch-graph-bc774-7778877fb9-g5794.18c2727d1017169e namespace: kserve-ci-e2e-test resourceVersion: "26508" uid: 887ddcb5-498d-479c-91de-d56b01d2723c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bc774} kind: Pod name: switch-graph-bc774-7778877fb9-g5794 namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 8fc0b3a5-fbf0-4cac-b4f8-be3a4d729942 kind: Event lastTimestamp: "2026-07-15T11:18:03Z" message: Started container switch-graph-bc774 metadata: creationTimestamp: "2026-07-15T11:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:18:03Z" name: switch-graph-bc774-7778877fb9-g5794.18c2727d108b37ce namespace: kserve-ci-e2e-test resourceVersion: "26509" uid: c053e977-bbda-4bbe-8d79-70713ca7df6d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bc774} kind: Pod name: switch-graph-bc774-7778877fb9-g5794 namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 8fc0b3a5-fbf0-4cac-b4f8-be3a4d729942 kind: Event lastTimestamp: "2026-07-15T11:34:22Z" message: Stopping container switch-graph-bc774 metadata: creationTimestamp: "2026-07-15T11:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:34:22Z" name: switch-graph-bc774-7778877fb9-g5794.18c27360e6a3367f namespace: kserve-ci-e2e-test resourceVersion: "33351" uid: f6c987ce-0d7d-48e2-a07f-23c0c39607b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:34:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bc774} kind: Pod name: switch-graph-bc774-7778877fb9-g5794 namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 8fc0b3a5-fbf0-4cac-b4f8-be3a4d729942 kind: Event lastTimestamp: "2026-07-15T11:34:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-15T11:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:34:50Z" name: switch-graph-bc774-7778877fb9-g5794.18c27361a9ed74f3 namespace: kserve-ci-e2e-test resourceVersion: "33627" uid: 08cf7d40-63c1-40e7-bc14-3f8739636b30 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-bc774-7778877fb9 namespace: kserve-ci-e2e-test resourceVersion: "26469" uid: 32013a87-6d85-4ebd-85b6-fa287c84d79e kind: Event lastTimestamp: "2026-07-15T11:18:03Z" message: 'Created pod: switch-graph-bc774-7778877fb9-g5794' metadata: creationTimestamp: "2026-07-15T11:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:03Z" name: switch-graph-bc774-7778877fb9.18c2727ceedab0cc namespace: kserve-ci-e2e-test resourceVersion: "26474" uid: 808c2954-8f70-4b47-9318-7c84d0d73d13 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-bc774 namespace: kserve-ci-e2e-test resourceVersion: "26466" uid: 7a74363e-177b-4d73-8559-c7064bfcc589 kind: Event lastTimestamp: "2026-07-15T11:18:03Z" message: Scaled up replica set switch-graph-bc774-7778877fb9 from 0 to 1 metadata: creationTimestamp: "2026-07-15T11:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:03Z" name: switch-graph-bc774.18c2727cee225d04 namespace: kserve-ci-e2e-test resourceVersion: "26471" uid: 9dd82506-56a1-449d-b187-2cdcda354911 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:18:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-bc774 namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: c9c05e5a-4fd2-48ee-b100-4360b3857184 kind: Event lastTimestamp: "2026-07-15T11:18:03Z" message: InferenceGraph [switch-graph-bc774] is Ready metadata: creationTimestamp: "2026-07-15T11:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T11:18:03Z" name: switch-graph-bc774.18c2727cf3620294 namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: fe899b8c-418d-480f-9f41-6b4a12d3f878 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:18:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-bc774 namespace: kserve-ci-e2e-test resourceVersion: "26468" uid: e667f205-e0ce-4c87-abc0-0bd41c72a071 kind: Event lastTimestamp: "2026-07-15T11:18:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:33Z" name: switch-graph-bc774.18c272806c97346f namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: 786fe709-b207-4ba3-a9d6-e1e0ad3193e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T11:18:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-bc774 namespace: kserve-ci-e2e-test resourceVersion: "26468" uid: e667f205-e0ce-4c87-abc0-0bd41c72a071 kind: Event lastTimestamp: "2026-07-15T11:18:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T11:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T11:18:33Z" name: switch-graph-bc774.18c272806cdbc635 namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: 18f35cad-0b4f-43c0-8fff-289dcc432722 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-15T10:57:33.239687Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ff4f8-fc4884fc4-c9572 namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: cd3e32df-59f4-4a0e-86b6-4e707c70d606 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-ff4f8-fc4884fc4-c9572 to ip-10-0-130-140.ec2.internal metadata: creationTimestamp: "2026-07-15T10:57:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-15T10:57:33Z" name: switch-graph-ff4f8-fc4884fc4-c9572.18c2715e897d4c28 namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: c95cdca8-26d2-48a1-950d-9a71ceefdaee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-79c4cd6986-jgcv6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:33Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ff4f8-fc4884fc4-c9572 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: cd3e32df-59f4-4a0e-86b6-4e707c70d606 kind: Event lastTimestamp: "2026-07-15T10:57:33Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-15T10:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-15T10:57:33Z" name: switch-graph-ff4f8-fc4884fc4-c9572.18c2715ea37fb4ce namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 6fab0b56-7839-45a1-a4c5-8eff43ae4e9a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ff4f8} kind: Pod name: switch-graph-ff4f8-fc4884fc4-c9572 namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: cd3e32df-59f4-4a0e-86b6-4e707c70d606 kind: Event lastTimestamp: "2026-07-15T10:57:33Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0e1a169e211c4cdcd19b5d9ae2b04ed590c1bcb2f57424f5c6e7d0d0f2faa1f0" already present on machine metadata: creationTimestamp: "2026-07-15T10:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:33Z" name: switch-graph-ff4f8-fc4884fc4-c9572.18c2715ea4e3c5f3 namespace: kserve-ci-e2e-test resourceVersion: "16045" uid: eeebec28-ff88-43dc-84c0-70119887f9d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ff4f8} kind: Pod name: switch-graph-ff4f8-fc4884fc4-c9572 namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: cd3e32df-59f4-4a0e-86b6-4e707c70d606 kind: Event lastTimestamp: "2026-07-15T10:57:33Z" message: 'Created container: switch-graph-ff4f8' metadata: creationTimestamp: "2026-07-15T10:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:33Z" name: switch-graph-ff4f8-fc4884fc4-c9572.18c2715ea8556025 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: a1f4fa16-8535-40e8-b235-e0ef92c50eac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ff4f8} kind: Pod name: switch-graph-ff4f8-fc4884fc4-c9572 namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: cd3e32df-59f4-4a0e-86b6-4e707c70d606 kind: Event lastTimestamp: "2026-07-15T10:57:33Z" message: Started container switch-graph-ff4f8 metadata: creationTimestamp: "2026-07-15T10:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T10:57:33Z" name: switch-graph-ff4f8-fc4884fc4-c9572.18c2715ea8caef14 namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 1d9e409a-0c86-4d70-9a95-61593eeee8ee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T11:05:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ff4f8} kind: Pod name: switch-graph-ff4f8-fc4884fc4-c9572 namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: cd3e32df-59f4-4a0e-86b6-4e707c70d606 kind: Event lastTimestamp: "2026-07-15T11:05:47Z" message: Stopping container switch-graph-ff4f8 metadata: creationTimestamp: "2026-07-15T11:05:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:05:47Z" name: switch-graph-ff4f8-fc4884fc4-c9572.18c271d1ace9949c namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: b498d937-1dbc-4c0a-8864-d472aa470d01 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-15T11:05:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ff4f8} kind: Pod name: switch-graph-ff4f8-fc4884fc4-c9572 namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: cd3e32df-59f4-4a0e-86b6-4e707c70d606 kind: Event lastTimestamp: "2026-07-15T11:06:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-15T11:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-15T11:06:14Z" name: switch-graph-ff4f8-fc4884fc4-c9572.18c271d22fb8948e namespace: kserve-ci-e2e-test resourceVersion: "19868" uid: a66040ec-7d4d-407c-af2b-fd351b117c68 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-140.ec2.internal source: component: kubelet host: ip-10-0-130-140.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-ff4f8-fc4884fc4 namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: a329a3c1-2605-4be5-853b-6f38060f01ec kind: Event lastTimestamp: "2026-07-15T10:57:33Z" message: 'Created pod: switch-graph-ff4f8-fc4884fc4-c9572' metadata: creationTimestamp: "2026-07-15T10:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:33Z" name: switch-graph-ff4f8-fc4884fc4.18c2715e88a90abc namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: 0fe46806-7a0a-4c17-b8fd-a0b9e3bb0e3c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-ff4f8 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 41714eb0-2777-42b3-a09b-437b584383c0 kind: Event lastTimestamp: "2026-07-15T10:57:33Z" message: Scaled up replica set switch-graph-ff4f8-fc4884fc4 from 0 to 1 metadata: creationTimestamp: "2026-07-15T10:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:57:33Z" name: switch-graph-ff4f8.18c2715e875229f1 namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: 4be86724-fa5c-4bac-bb98-9d8261ae00b7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-15T10:57:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ff4f8 namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: 3cfd8377-16ab-448b-ac76-abc676af7dc7 kind: Event lastTimestamp: "2026-07-15T10:57:33Z" message: InferenceGraph [switch-graph-ff4f8] is Ready metadata: creationTimestamp: "2026-07-15T10:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-15T10:57:33Z" name: switch-graph-ff4f8.18c2715e8e1e6c76 namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: c8bb1dce-dbb1-44e8-86c4-eaaa2518ce03 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T10:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ff4f8 namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: 729c70c2-d939-48b5-b30c-9b4159261c97 kind: Event lastTimestamp: "2026-07-15T10:58:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:58:03Z" name: switch-graph-ff4f8.18c2716205bae79c namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 3c6f4a73-7b4a-4df2-ba14-333d720c527c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-15T10:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ff4f8 namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: 729c70c2-d939-48b5-b30c-9b4159261c97 kind: Event lastTimestamp: "2026-07-15T10:58:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-15T10:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-15T10:58:03Z" name: switch-graph-ff4f8.18c271620627cc04 namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 2bd9d956-cca2-4e82-998a-68244b36bb8a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33667"