--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:40:19.348036Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-02ff4-854c9cfd46-slghq namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: e542f4f7-5fee-4226-b2f9-deccc433481d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-02ff4-854c9cfd46-slghq to ip-10-0-134-97.ec2.internal metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:40:19Z" name: ensemble-graph-02ff4-854c9cfd46-slghq.18c1f4029556f8de namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: f3d4c89d-2ce5-414c-9f3a-e028b5ee6940 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-02ff4-854c9cfd46-slghq namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: e542f4f7-5fee-4226-b2f9-deccc433481d kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-02ff4-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: ensemble-graph-02ff4-854c9cfd46-slghq.18c1f402a251d4cb namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: 602a9633-8b7f-45da-bfb1-7b02f39a54b6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:20Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-02ff4-854c9cfd46-slghq namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: e542f4f7-5fee-4226-b2f9-deccc433481d kind: Event lastTimestamp: "2026-07-13T20:40:20Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:40:20Z" name: ensemble-graph-02ff4-854c9cfd46-slghq.18c1f402d259c2e6 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: 1b579d8e-e432-429a-9f68-d4ecd34b289f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-02ff4} kind: Pod name: ensemble-graph-02ff4-854c9cfd46-slghq namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: e542f4f7-5fee-4226-b2f9-deccc433481d kind: Event lastTimestamp: "2026-07-13T20:40:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:8345cd1e0b8ad942d68e2fe65c99d692fac6452f8ea219efb6700aaf29b3e2fa" already present on machine metadata: creationTimestamp: "2026-07-13T20:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:20Z" name: ensemble-graph-02ff4-854c9cfd46-slghq.18c1f402d439d7a9 namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 996a51ef-6df5-4a43-ba88-d375c702d22c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-02ff4} kind: Pod name: ensemble-graph-02ff4-854c9cfd46-slghq namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: e542f4f7-5fee-4226-b2f9-deccc433481d kind: Event lastTimestamp: "2026-07-13T20:40:20Z" message: 'Created container: ensemble-graph-02ff4' metadata: creationTimestamp: "2026-07-13T20:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:20Z" name: ensemble-graph-02ff4-854c9cfd46-slghq.18c1f402d72602fa namespace: kserve-ci-e2e-test resourceVersion: "20339" uid: ea68a268-afc3-47a4-8f85-bd5ea0e6d77f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-02ff4} kind: Pod name: ensemble-graph-02ff4-854c9cfd46-slghq namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: e542f4f7-5fee-4226-b2f9-deccc433481d kind: Event lastTimestamp: "2026-07-13T20:40:20Z" message: Started container ensemble-graph-02ff4 metadata: creationTimestamp: "2026-07-13T20:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:20Z" name: ensemble-graph-02ff4-854c9cfd46-slghq.18c1f402d7f17183 namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 4ce9e8e0-460d-41e9-a072-9bf209c43a65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-02ff4} kind: Pod name: ensemble-graph-02ff4-854c9cfd46-slghq namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: e542f4f7-5fee-4226-b2f9-deccc433481d kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: Stopping container ensemble-graph-02ff4 metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:29Z" name: ensemble-graph-02ff4-854c9cfd46-slghq.18c1f404ed598e7d namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: bba02ae0-1da9-46e8-867d-d6bee46ca74b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:40:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-02ff4} kind: Pod name: ensemble-graph-02ff4-854c9cfd46-slghq namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: e542f4f7-5fee-4226-b2f9-deccc433481d kind: Event lastTimestamp: "2026-07-13T20:40:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T20:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:57Z" name: ensemble-graph-02ff4-854c9cfd46-slghq.18c1f4058be69f68 namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: e166fe49-b296-4602-9e13-8ef83d476d1a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-02ff4-854c9cfd46 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: 2e011c10-c3c1-4d55-b33d-c16f0e6e74d0 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: 'Created pod: ensemble-graph-02ff4-854c9cfd46-slghq' metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:19Z" name: ensemble-graph-02ff4-854c9cfd46.18c1f40294689ef2 namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: fa51f282-04d6-42a2-b856-9325ef88c4d1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-02ff4 namespace: kserve-ci-e2e-test resourceVersion: "20276" uid: 55571887-acc5-463d-8bf2-ef204256edc3 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Scaled up replica set ensemble-graph-02ff4-854c9cfd46 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:19Z" name: ensemble-graph-02ff4.18c1f402934f5a27 namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: 52a7391b-614a-41c5-8dcf-640c017eb0e7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-02ff4 namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: 126d96c5-47a2-45af-a0b5-29c0e11b113a kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: InferenceGraph [ensemble-graph-02ff4] is Ready metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:40:19Z" name: ensemble-graph-02ff4.18c1f4029ab5c009 namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 71cb79ec-6ff0-4b15-938e-9b5d560e67ac reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:41:29.712965Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-db27e-7896bb48fc-nhrvq namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: 6868ae1f-8f62-428d-949f-8d37a7f9148d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-db27e-7896bb48fc-nhrvq to ip-10-0-134-97.ec2.internal metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:41:29Z" name: ensemble-graph-db27e-7896bb48fc-nhrvq.18c1f412f76a9661 namespace: kserve-ci-e2e-test resourceVersion: "21258" uid: 886eb0f0-ff62-4ebb-b442-895188fc0a29 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-db27e-7896bb48fc-nhrvq namespace: kserve-ci-e2e-test resourceVersion: "21254" uid: 6868ae1f-8f62-428d-949f-8d37a7f9148d kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-db27e-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: ensemble-graph-db27e-7896bb48fc-nhrvq.18c1f41303a07856 namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: e29ddc35-3b72-4763-98ee-32d3fff8ccd1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:30Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-db27e-7896bb48fc-nhrvq namespace: kserve-ci-e2e-test resourceVersion: "21263" uid: 6868ae1f-8f62-428d-949f-8d37a7f9148d kind: Event lastTimestamp: "2026-07-13T20:41:30Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:41:30Z" name: ensemble-graph-db27e-7896bb48fc-nhrvq.18c1f41334735e51 namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: afe5d4c9-0b87-4c82-ac26-374be610674b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-db27e} kind: Pod name: ensemble-graph-db27e-7896bb48fc-nhrvq namespace: kserve-ci-e2e-test resourceVersion: "21254" uid: 6868ae1f-8f62-428d-949f-8d37a7f9148d kind: Event lastTimestamp: "2026-07-13T20:41:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:8345cd1e0b8ad942d68e2fe65c99d692fac6452f8ea219efb6700aaf29b3e2fa" already present on machine metadata: creationTimestamp: "2026-07-13T20:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:30Z" name: ensemble-graph-db27e-7896bb48fc-nhrvq.18c1f41335d459b0 namespace: kserve-ci-e2e-test resourceVersion: "21312" uid: 2b0103ca-e011-47c1-9bdc-2310e920c755 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-db27e} kind: Pod name: ensemble-graph-db27e-7896bb48fc-nhrvq namespace: kserve-ci-e2e-test resourceVersion: "21254" uid: 6868ae1f-8f62-428d-949f-8d37a7f9148d kind: Event lastTimestamp: "2026-07-13T20:41:30Z" message: 'Created container: ensemble-graph-db27e' metadata: creationTimestamp: "2026-07-13T20:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:30Z" name: ensemble-graph-db27e-7896bb48fc-nhrvq.18c1f41339055126 namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: 05e82ebf-950c-4d22-beaa-9367f56fc202 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-db27e} kind: Pod name: ensemble-graph-db27e-7896bb48fc-nhrvq namespace: kserve-ci-e2e-test resourceVersion: "21254" uid: 6868ae1f-8f62-428d-949f-8d37a7f9148d kind: Event lastTimestamp: "2026-07-13T20:41:30Z" message: Started container ensemble-graph-db27e metadata: creationTimestamp: "2026-07-13T20:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:30Z" name: ensemble-graph-db27e-7896bb48fc-nhrvq.18c1f41339920646 namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: 59dab938-9704-48c7-80aa-f2b7df745a1d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-db27e} kind: Pod name: ensemble-graph-db27e-7896bb48fc-nhrvq namespace: kserve-ci-e2e-test resourceVersion: "21254" uid: 6868ae1f-8f62-428d-949f-8d37a7f9148d kind: Event lastTimestamp: "2026-07-13T20:49:44Z" message: Stopping container ensemble-graph-db27e metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:44Z" name: ensemble-graph-db27e-7896bb48fc-nhrvq.18c1f4862259962e namespace: kserve-ci-e2e-test resourceVersion: "24794" uid: c63ea281-0bda-4b50-8924-28a6e2ea6318 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:49:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-db27e} kind: Pod name: ensemble-graph-db27e-7896bb48fc-nhrvq namespace: kserve-ci-e2e-test resourceVersion: "21254" uid: 6868ae1f-8f62-428d-949f-8d37a7f9148d kind: Event lastTimestamp: "2026-07-13T20:50:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T20:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:12Z" name: ensemble-graph-db27e-7896bb48fc-nhrvq.18c1f486d572f408 namespace: kserve-ci-e2e-test resourceVersion: "25137" uid: ef6e79ed-9971-4818-ad05-4d6a84a4af7f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-db27e-7896bb48fc namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 022114cf-04b6-4ff0-afa4-26be4e2e368e kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: 'Created pod: ensemble-graph-db27e-7896bb48fc-nhrvq' metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:29Z" name: ensemble-graph-db27e-7896bb48fc.18c1f412f6de923f namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: af3b711c-a4ca-4697-9d0c-471a7348b0ac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-db27e namespace: kserve-ci-e2e-test resourceVersion: "21248" uid: 2eda0261-73c1-4851-831e-e2fe7d85babf kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Scaled up replica set ensemble-graph-db27e-7896bb48fc from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:29Z" name: ensemble-graph-db27e.18c1f412f598bc20 namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: 7bd3e109-a198-47fe-9e3f-291fb8863f71 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-db27e namespace: kserve-ci-e2e-test resourceVersion: "21267" uid: 1d103a7d-ad7b-4908-a9d5-8b8710c23465 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: InferenceGraph [ensemble-graph-db27e] is Ready metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:29Z" name: ensemble-graph-db27e.18c1f412fc7c8013 namespace: kserve-ci-e2e-test resourceVersion: "21271" uid: eb9baf9e-cb88-4239-855e-6c44fdb269fc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:41:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-db27e namespace: kserve-ci-e2e-test resourceVersion: "21249" uid: 66fdfe8c-4abc-42e8-8c89-34676c4cfa68 kind: Event lastTimestamp: "2026-07-13T20:41:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:59Z" name: ensemble-graph-db27e.18c1f41674207190 namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: 823646f4-03c0-4e0a-9d6a-317c14bf7dba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:41:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-db27e namespace: kserve-ci-e2e-test resourceVersion: "21249" uid: 66fdfe8c-4abc-42e8-8c89-34676c4cfa68 kind: Event lastTimestamp: "2026-07-13T20:41:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:59Z" name: ensemble-graph-db27e.18c1f416748a4921 namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: 31da995a-eb5b-4bb7-8907-295d44e86b85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:39:19.382556Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk namespace: kserve-ci-e2e-test resourceVersion: "19634" uid: 45429bb1-d02d-41de-815c-9aab4b42fb9f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-02ff4-predictor-7748bf4588-2k6pk to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:39:19Z" name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk.18c1f3f49f1e51da namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: e65caf5f-e390-4bc3-b118-bd20e01d128b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 45429bb1-d02d-41de-815c-9aab4b42fb9f kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-02ff4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:19Z" name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk.18c1f3f4aa5d4621 namespace: kserve-ci-e2e-test resourceVersion: "19664" uid: 92dec754-da9f-4311-ab8c-1f9271e763ac reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk namespace: kserve-ci-e2e-test resourceVersion: "19649" uid: 45429bb1-d02d-41de-815c-9aab4b42fb9f kind: Event lastTimestamp: "2026-07-13T20:39:20Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:39:20Z" name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk.18c1f3f4dc67210f namespace: kserve-ci-e2e-test resourceVersion: "19688" uid: c74bb229-1b65-424c-800d-c6b041c4979e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 45429bb1-d02d-41de-815c-9aab4b42fb9f kind: Event lastTimestamp: "2026-07-13T20:39:20Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:20Z" name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk.18c1f3f4de1f2f97 namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 9c617adc-af5d-4e3f-9b17-dbdb35171b8c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 45429bb1-d02d-41de-815c-9aab4b42fb9f kind: Event lastTimestamp: "2026-07-13T20:39:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:20Z" name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk.18c1f3f4e11f5c15 namespace: kserve-ci-e2e-test resourceVersion: "19692" uid: 749a64f6-c4ac-40cb-9fec-ec1a452bd52f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 45429bb1-d02d-41de-815c-9aab4b42fb9f kind: Event lastTimestamp: "2026-07-13T20:39:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:20Z" name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk.18c1f3f4e1974251 namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 348281a1-7a7b-428c-a9cf-66a4c0f78f18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 45429bb1-d02d-41de-815c-9aab4b42fb9f kind: Event lastTimestamp: "2026-07-13T20:39:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:20Z" name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk.18c1f3f4e1a12668 namespace: kserve-ci-e2e-test resourceVersion: "19694" uid: c40df3cd-be7e-4047-866b-d31e71f2f230 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 45429bb1-d02d-41de-815c-9aab4b42fb9f kind: Event lastTimestamp: "2026-07-13T20:39:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:20Z" name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk.18c1f3f4e4b35920 namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: 642559f8-307d-4e6d-8a11-56a8dfc30244 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 45429bb1-d02d-41de-815c-9aab4b42fb9f kind: Event lastTimestamp: "2026-07-13T20:39:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:20Z" name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk.18c1f3f4e52d9ad6 namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: 4780eb9c-0d64-4789-81d2-767ba5a16a7c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:39:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 45429bb1-d02d-41de-815c-9aab4b42fb9f kind: Event lastTimestamp: "2026-07-13T20:39:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:57Z" name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk.18c1f3f5346ecc67 namespace: kserve-ci-e2e-test resourceVersion: "19993" uid: a8ac3ce2-7480-4a76-9d3a-7e125b930ea1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 45429bb1-d02d-41de-815c-9aab4b42fb9f kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:29Z" name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk.18c1f404fe265876 namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: f76a2545-683d-4f83-ba47-8cd48aa8854c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 45429bb1-d02d-41de-815c-9aab4b42fb9f kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:29Z" name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk.18c1f404fe26cfca namespace: kserve-ci-e2e-test resourceVersion: "20518" uid: 17dcdca2-0b4b-4b10-85f4-3252451cc7c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 45429bb1-d02d-41de-815c-9aab4b42fb9f kind: Event lastTimestamp: "2026-07-13T20:40:32Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:32Z" name: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk.18c1f405bcddb0a2 namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: 6eda8ff8-dd6d-4108-9f26-f393e85458c4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-02ff4-predictor-7748bf4588 namespace: kserve-ci-e2e-test resourceVersion: "19626" uid: fa787e75-fa3d-4022-836c-e4221306e55e kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: 'Created pod: error-404-isvc-02ff4-predictor-7748bf4588-2k6pk' metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:39:19Z" name: error-404-isvc-02ff4-predictor-7748bf4588.18c1f3f49e6955b5 namespace: kserve-ci-e2e-test resourceVersion: "19636" uid: 90bc0e71-457e-4bce-8333-6312b7a42faa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-02ff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19621" uid: 836f1201-482b-416b-92b0-abf95834e879 kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: Scaled up replica set error-404-isvc-02ff4-predictor-7748bf4588 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:39:19Z" name: error-404-isvc-02ff4-predictor.18c1f3f49bc2eec3 namespace: kserve-ci-e2e-test resourceVersion: "19629" uid: 0b9a62eb-cb36-4c0c-89eb-a88f143f1a90 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-02ff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19624" uid: c89f50e5-c874-4517-9d92-34f0e3f030af kind: Event lastTimestamp: "2026-07-13T20:39:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:39:34Z" name: error-404-isvc-02ff4-predictor.18c1f3f81a29a3e8 namespace: kserve-ci-e2e-test resourceVersion: "19818" uid: 05511ec1-1c91-458b-827e-7410446734fa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-02ff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19624" uid: c89f50e5-c874-4517-9d92-34f0e3f030af kind: Event lastTimestamp: "2026-07-13T20:39:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:39:34Z" name: error-404-isvc-02ff4-predictor.18c1f3f81aa38cff namespace: kserve-ci-e2e-test resourceVersion: "19820" uid: 6b2b42b8-636c-465d-a306-385a4494f6ba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:39:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-02ff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19819" uid: c89f50e5-c874-4517-9d92-34f0e3f030af kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-02ff4-predictor.18c1f3fb999e619c namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: a3ed4cff-3913-4904-a650-9ab3d3230d3b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:39:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-02ff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19819" uid: c89f50e5-c874-4517-9d92-34f0e3f030af kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-02ff4-predictor.18c1f3fb9a1701bb namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: 8c908bdc-7e8f-4593-82d1-799d3032c9f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-02ff4 namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: 484c674d-3aea-4fe7-b0c8-9351db62c8ea kind: Event lastTimestamp: "2026-07-13T20:40:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:40:08Z" name: error-404-isvc-02ff4.18c1f3f49bc54678 namespace: kserve-ci-e2e-test resourceVersion: "20084" uid: daf90996-dad1-4235-af46-1d9efb7cef1d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-02ff4 namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: 484c674d-3aea-4fe7-b0c8-9351db62c8ea kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: 'Failed to update status for InferenceService "error-404-isvc-02ff4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-02ff4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:39:19Z" name: error-404-isvc-02ff4.18c1f3f49cdbd19e namespace: kserve-ci-e2e-test resourceVersion: "19631" uid: d0c91a64-8d15-4275-81bd-292623d94f24 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-02ff4 namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: 484c674d-3aea-4fe7-b0c8-9351db62c8ea kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-02ff4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:39:19Z" name: error-404-isvc-02ff4.18c1f3f49cdc2fa0 namespace: kserve-ci-e2e-test resourceVersion: "19632" uid: 4e72d677-278a-4b04-8484-76071734cd35 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-02ff4 namespace: kserve-ci-e2e-test resourceVersion: "20082" uid: 484c674d-3aea-4fe7-b0c8-9351db62c8ea kind: Event lastTimestamp: "2026-07-13T20:40:08Z" message: InferenceService [error-404-isvc-02ff4] is Ready metadata: creationTimestamp: "2026-07-13T20:40:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:40:08Z" name: error-404-isvc-02ff4.18c1f3fff1b5fb6d namespace: kserve-ci-e2e-test resourceVersion: "20083" uid: f6858159-f3e6-4358-9944-e8a90843400f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:49:44.707057Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx namespace: kserve-ci-e2e-test resourceVersion: "24876" uid: 6154d545-13b4-434f-8c60-9799415ee4ef kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-35019-predictor-86b8b845bd-2l9hx to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:49:44Z" name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx.18c1f486375d0c34 namespace: kserve-ci-e2e-test resourceVersion: "24882" uid: e55beec8-e56f-4bb0-b800-629b3ea8bd14 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx namespace: kserve-ci-e2e-test resourceVersion: "24886" uid: 6154d545-13b4-434f-8c60-9799415ee4ef kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:49:45Z" name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx.18c1f48651c9a374 namespace: kserve-ci-e2e-test resourceVersion: "24905" uid: ccb4d370-9201-431c-a051-1bf4c961445e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx namespace: kserve-ci-e2e-test resourceVersion: "24878" uid: 6154d545-13b4-434f-8c60-9799415ee4ef kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:45Z" name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx.18c1f486539baef1 namespace: kserve-ci-e2e-test resourceVersion: "24907" uid: d1800c8d-5f53-46d8-8de6-984b7fd0a121 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx namespace: kserve-ci-e2e-test resourceVersion: "24878" uid: 6154d545-13b4-434f-8c60-9799415ee4ef kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:45Z" name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx.18c1f48655def7fc namespace: kserve-ci-e2e-test resourceVersion: "24910" uid: 3f6d997a-f29b-4a55-b4c8-504e15380328 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx namespace: kserve-ci-e2e-test resourceVersion: "24878" uid: 6154d545-13b4-434f-8c60-9799415ee4ef kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:45Z" name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx.18c1f48656538f19 namespace: kserve-ci-e2e-test resourceVersion: "24911" uid: da870eed-6bbc-43f0-b51e-11deffa13e64 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx namespace: kserve-ci-e2e-test resourceVersion: "24878" uid: 6154d545-13b4-434f-8c60-9799415ee4ef kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:45Z" name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx.18c1f486565d9314 namespace: kserve-ci-e2e-test resourceVersion: "24912" uid: ef3a74f9-e0e6-40be-8bb3-ab56382d7c3b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx namespace: kserve-ci-e2e-test resourceVersion: "24878" uid: 6154d545-13b4-434f-8c60-9799415ee4ef kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:45Z" name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx.18c1f4865a56fecf namespace: kserve-ci-e2e-test resourceVersion: "24913" uid: 3d294a28-95ab-409b-826c-1154b589b68d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx namespace: kserve-ci-e2e-test resourceVersion: "24878" uid: 6154d545-13b4-434f-8c60-9799415ee4ef kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:45Z" name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx.18c1f4865ad17577 namespace: kserve-ci-e2e-test resourceVersion: "24914" uid: 56c28959-05c7-4282-b8a2-7dddf31db76e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx namespace: kserve-ci-e2e-test resourceVersion: "24878" uid: 6154d545-13b4-434f-8c60-9799415ee4ef kind: Event lastTimestamp: "2026-07-13T20:50:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:21Z" name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx.18c1f48665ba936e namespace: kserve-ci-e2e-test resourceVersion: "25377" uid: 892e98c2-082e-4bef-be3a-bfbe63af5d72 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx namespace: kserve-ci-e2e-test resourceVersion: "24878" uid: 6154d545-13b4-434f-8c60-9799415ee4ef kind: Event lastTimestamp: "2026-07-13T20:50:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:54Z" name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx.18c1f4968afdd71f namespace: kserve-ci-e2e-test resourceVersion: "26022" uid: a36d4eaf-7057-44b4-9f05-18e57c44c27a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx namespace: kserve-ci-e2e-test resourceVersion: "24878" uid: 6154d545-13b4-434f-8c60-9799415ee4ef kind: Event lastTimestamp: "2026-07-13T20:50:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:54Z" name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx.18c1f4968afe5526 namespace: kserve-ci-e2e-test resourceVersion: "26025" uid: cdf7f91d-108c-4bae-b5b5-66c97a4d5803 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx namespace: kserve-ci-e2e-test resourceVersion: "24878" uid: 6154d545-13b4-434f-8c60-9799415ee4ef kind: Event lastTimestamp: "2026-07-13T20:50:56Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:56Z" name: error-404-isvc-35019-predictor-86b8b845bd-2l9hx.18c1f496ee4b7803 namespace: kserve-ci-e2e-test resourceVersion: "26107" uid: c6e06601-9717-4a67-a8b4-6c1f451e82bb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-35019-predictor-86b8b845bd namespace: kserve-ci-e2e-test resourceVersion: "24857" uid: 891c4762-efc4-47dd-addd-871d395afcfd kind: Event lastTimestamp: "2026-07-13T20:49:44Z" message: 'Created pod: error-404-isvc-35019-predictor-86b8b845bd-2l9hx' metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:49:44Z" name: error-404-isvc-35019-predictor-86b8b845bd.18c1f48636d87183 namespace: kserve-ci-e2e-test resourceVersion: "24879" uid: 339ce34f-9374-4351-ac7d-810f0af04265 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-35019-predictor namespace: kserve-ci-e2e-test resourceVersion: "24851" uid: 32bc9bbd-c1dc-46f8-84d8-66d2316ef447 kind: Event lastTimestamp: "2026-07-13T20:49:44Z" message: Scaled up replica set error-404-isvc-35019-predictor-86b8b845bd from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:49:44Z" name: error-404-isvc-35019-predictor.18c1f48632f9a88c namespace: kserve-ci-e2e-test resourceVersion: "24865" uid: d4383636-d988-4e21-8536-cf454b70e085 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:49:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-35019-predictor namespace: kserve-ci-e2e-test resourceVersion: "24855" uid: 4104e8eb-d5b0-4041-8587-783555361892 kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:44Z" name: error-404-isvc-35019-predictor.18c1f489b1618f87 namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: b2b5d018-1f07-4f89-a662-fa2a2f13fad1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:49:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-35019-predictor namespace: kserve-ci-e2e-test resourceVersion: "24855" uid: 4104e8eb-d5b0-4041-8587-783555361892 kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:44Z" name: error-404-isvc-35019-predictor.18c1f489b1c8765a namespace: kserve-ci-e2e-test resourceVersion: "25828" uid: 17d9ad49-8386-4f36-9659-309cd3a112a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-35019-predictor namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: 4104e8eb-d5b0-4041-8587-783555361892 kind: Event lastTimestamp: "2026-07-13T20:50:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:14Z" name: error-404-isvc-35019-predictor.18c1f48d3077a713 namespace: kserve-ci-e2e-test resourceVersion: "25158" uid: 19fe7e3e-685a-4f14-a9a0-3f098f477ffc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-35019-predictor namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: 4104e8eb-d5b0-4041-8587-783555361892 kind: Event lastTimestamp: "2026-07-13T20:50:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:14Z" name: error-404-isvc-35019-predictor.18c1f48d30dcb7c0 namespace: kserve-ci-e2e-test resourceVersion: "25160" uid: 6f49801c-d9d9-4ca7-9429-3002f512bd43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-35019 namespace: kserve-ci-e2e-test resourceVersion: "24834" uid: 37b71220-bd5e-4022-b7b4-3f9583cf0579 kind: Event lastTimestamp: "2026-07-13T20:50:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:50:31Z" name: error-404-isvc-35019.18c1f4863391b4c2 namespace: kserve-ci-e2e-test resourceVersion: "25577" uid: 79e38dc9-29fe-4fdd-98b0-addb2e647e0c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-35019 namespace: kserve-ci-e2e-test resourceVersion: "24834" uid: 37b71220-bd5e-4022-b7b4-3f9583cf0579 kind: Event lastTimestamp: "2026-07-13T20:49:44Z" message: 'Failed to update status for InferenceService "error-404-isvc-35019": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-35019": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:49:44Z" name: error-404-isvc-35019.18c1f48634caae69 namespace: kserve-ci-e2e-test resourceVersion: "24869" uid: 4df2a546-935a-4a3c-9a24-51724966ce59 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-35019 namespace: kserve-ci-e2e-test resourceVersion: "24834" uid: 37b71220-bd5e-4022-b7b4-3f9583cf0579 kind: Event lastTimestamp: "2026-07-13T20:49:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-35019": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:49:44Z" name: error-404-isvc-35019.18c1f48634cb1fcf namespace: kserve-ci-e2e-test resourceVersion: "24871" uid: be61ab07-b36c-47a9-8926-a7ba207f0cba reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-35019 namespace: kserve-ci-e2e-test resourceVersion: "25575" uid: 37b71220-bd5e-4022-b7b4-3f9583cf0579 kind: Event lastTimestamp: "2026-07-13T20:50:31Z" message: InferenceService [error-404-isvc-35019] is Ready metadata: creationTimestamp: "2026-07-13T20:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:50:31Z" name: error-404-isvc-35019.18c1f49121438dc5 namespace: kserve-ci-e2e-test resourceVersion: "25576" uid: 76cdc33c-6dad-4c67-8f81-5a2d378a5d01 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:31:04.276315Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-43619-predictor-8695456798-vdxlr namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: 35b0ae30-4d80-4bb6-b7e9-c071481aba95 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-43619-predictor-8695456798-vdxlr to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:31:04Z" name: error-404-isvc-43619-predictor-8695456798-vdxlr.18c1f381587cb4de namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 19873cf4-1af4-465d-8366-7d8f10568182 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-43619-predictor-8695456798-vdxlr namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 35b0ae30-4d80-4bb6-b7e9-c071481aba95 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-43619-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:04Z" name: error-404-isvc-43619-predictor-8695456798-vdxlr.18c1f38163b5b68a namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 5d4a14ea-89cd-4cec-8c1a-7ba1be2fc5c1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-43619-predictor-8695456798-vdxlr namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: 35b0ae30-4d80-4bb6-b7e9-c071481aba95 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:31:05Z" name: error-404-isvc-43619-predictor-8695456798-vdxlr.18c1f38195eb9591 namespace: kserve-ci-e2e-test resourceVersion: "16050" uid: 7a82105f-5f5b-4c1c-97e3-2ccff5e21a76 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43619-predictor-8695456798-vdxlr namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 35b0ae30-4d80-4bb6-b7e9-c071481aba95 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:05Z" name: error-404-isvc-43619-predictor-8695456798-vdxlr.18c1f381979b8fd0 namespace: kserve-ci-e2e-test resourceVersion: "16055" uid: 3094035f-2769-458f-b93e-cd8ac3095d2f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43619-predictor-8695456798-vdxlr namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 35b0ae30-4d80-4bb6-b7e9-c071481aba95 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:05Z" name: error-404-isvc-43619-predictor-8695456798-vdxlr.18c1f3819a2cf85d namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: 8e8482c0-551b-436a-998f-6f23d8b2c4cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43619-predictor-8695456798-vdxlr namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 35b0ae30-4d80-4bb6-b7e9-c071481aba95 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:05Z" name: error-404-isvc-43619-predictor-8695456798-vdxlr.18c1f3819acb5ec1 namespace: kserve-ci-e2e-test resourceVersion: "16060" uid: e6ec8ad2-2a2d-467a-bf65-d718f4a6bfa4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-43619-predictor-8695456798-vdxlr namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 35b0ae30-4d80-4bb6-b7e9-c071481aba95 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:05Z" name: error-404-isvc-43619-predictor-8695456798-vdxlr.18c1f3819ad6927d namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 2b98d7c5-14f3-4fab-ba13-c822d01e6d81 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-43619-predictor-8695456798-vdxlr namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 35b0ae30-4d80-4bb6-b7e9-c071481aba95 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:05Z" name: error-404-isvc-43619-predictor-8695456798-vdxlr.18c1f3819dcf6e0f namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 001eeb58-356f-4916-8522-805fa38d6929 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-43619-predictor-8695456798-vdxlr namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 35b0ae30-4d80-4bb6-b7e9-c071481aba95 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:05Z" name: error-404-isvc-43619-predictor-8695456798-vdxlr.18c1f3819e4482ed namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 40aca3e3-8c1e-4c97-a41c-f22d404f5954 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:31:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43619-predictor-8695456798-vdxlr namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 35b0ae30-4d80-4bb6-b7e9-c071481aba95 kind: Event lastTimestamp: "2026-07-13T20:31:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:42Z" name: error-404-isvc-43619-predictor-8695456798-vdxlr.18c1f381f3e678b1 namespace: kserve-ci-e2e-test resourceVersion: "16386" uid: bc11ad19-f925-4869-81d8-e25fb17f4fa9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43619-predictor-8695456798-vdxlr namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 35b0ae30-4d80-4bb6-b7e9-c071481aba95 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-43619-predictor-8695456798-vdxlr.18c1f4028629587a namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: ed0ae3e4-d649-483b-be7a-a4433b4e0b69 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-43619-predictor-8695456798-vdxlr namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 35b0ae30-4d80-4bb6-b7e9-c071481aba95 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-43619-predictor-8695456798-vdxlr.18c1f4028629d340 namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: 35cc4513-56c0-4a3f-ac62-2f2135ebc4c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-43619-predictor-8695456798 namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: 058fa0c5-7a58-4986-8ba1-5827281d30ca kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: 'Created pod: error-404-isvc-43619-predictor-8695456798-vdxlr' metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:04Z" name: error-404-isvc-43619-predictor-8695456798.18c1f381580718a2 namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: f653f0cb-fe0b-4d9e-973f-535962ac578e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-43619-predictor namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: 319538ce-0440-416d-bd46-251fbf85e796 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: Scaled up replica set error-404-isvc-43619-predictor-8695456798 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:04Z" name: error-404-isvc-43619-predictor.18c1f38155e1c346 namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 35eac106-afcd-41af-a994-8127725bf0aa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:31:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-43619-predictor namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: aa01c21f-c4f5-46df-b40a-3c813b877f2c kind: Event lastTimestamp: "2026-07-13T20:31:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:31:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:34Z" name: error-404-isvc-43619-predictor.18c1f384d44eaa01 namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 85fb772e-fbc2-488f-bfd9-d0d34cb95300 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:31:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-43619-predictor namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: aa01c21f-c4f5-46df-b40a-3c813b877f2c kind: Event lastTimestamp: "2026-07-13T20:31:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:31:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:34Z" name: error-404-isvc-43619-predictor.18c1f384d4c8af30 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 66e9981d-66d1-4e9a-8b0c-c0b2457a43bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:31:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-43619-predictor namespace: kserve-ci-e2e-test resourceVersion: "16202" uid: aa01c21f-c4f5-46df-b40a-3c813b877f2c kind: Event lastTimestamp: "2026-07-13T20:32:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:31:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:32:04Z" name: error-404-isvc-43619-predictor.18c1f38bd27eba13 namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: 8b23e87c-624e-4292-882f-db73d52df746 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:31:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-43619-predictor namespace: kserve-ci-e2e-test resourceVersion: "16202" uid: aa01c21f-c4f5-46df-b40a-3c813b877f2c kind: Event lastTimestamp: "2026-07-13T20:32:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:31:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:32:04Z" name: error-404-isvc-43619-predictor.18c1f38bd2d258db namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: e0ab2625-99c6-4615-8a2a-6aae2db28686 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-43619 namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: 340e486c-0900-4134-ae9b-94b81967b544 kind: Event lastTimestamp: "2026-07-13T20:31:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:31:53Z" name: error-404-isvc-43619.18c1f38155e966b3 namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: a0ce45ef-d72a-4008-8b63-e9f3bd0661e2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-43619 namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: 340e486c-0900-4134-ae9b-94b81967b544 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-43619": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-43619": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:31:04Z" name: error-404-isvc-43619.18c1f38156e03fd6 namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: c52dcab6-68ca-47e5-9575-92533ead7077 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-43619 namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: 340e486c-0900-4134-ae9b-94b81967b544 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-43619": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:31:04Z" name: error-404-isvc-43619.18c1f38156e077f0 namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: 96b00b67-0583-4475-a2d9-0085e70936d0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-43619 namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 340e486c-0900-4134-ae9b-94b81967b544 kind: Event lastTimestamp: "2026-07-13T20:31:53Z" message: InferenceService [error-404-isvc-43619] is Ready metadata: creationTimestamp: "2026-07-13T20:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:31:53Z" name: error-404-isvc-43619.18c1f38cb337c12e namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 10e6acd6-dd70-4732-a271-6e61db6fc8f2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:50:54.946524Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8 namespace: kserve-ci-e2e-test resourceVersion: "26054" uid: 63603a40-71b1-470e-bc4c-424e7c0f418c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-492f6-predictor-598d856bcd-kxgf8 to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:50:54Z" name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8.18c1f49691f6309a namespace: kserve-ci-e2e-test resourceVersion: "26058" uid: fea8edf0-a3f2-4a0d-b818-583ca8b7e241 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8 namespace: kserve-ci-e2e-test resourceVersion: "26065" uid: 63603a40-71b1-470e-bc4c-424e7c0f418c kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:50:55Z" name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8.18c1f496ac4b9179 namespace: kserve-ci-e2e-test resourceVersion: "26081" uid: a0455a9a-9f55-46a0-8f5d-218e372ae517 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 63603a40-71b1-470e-bc4c-424e7c0f418c kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:55Z" name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8.18c1f496adfdc0ce namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: c1bd1155-a1a0-42da-a873-1fdd5e91e785 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 63603a40-71b1-470e-bc4c-424e7c0f418c kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:55Z" name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8.18c1f496b0801fd1 namespace: kserve-ci-e2e-test resourceVersion: "26088" uid: aff2a318-d603-4c91-9a33-e796b4939834 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 63603a40-71b1-470e-bc4c-424e7c0f418c kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:55Z" name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8.18c1f496b10f2175 namespace: kserve-ci-e2e-test resourceVersion: "26090" uid: 6a452678-93a9-4197-9acb-20d086a88bc3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 63603a40-71b1-470e-bc4c-424e7c0f418c kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:55Z" name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8.18c1f496b1176ddf namespace: kserve-ci-e2e-test resourceVersion: "26091" uid: 39d4b6ed-56d0-48b6-867d-5da2c17b0dd9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 63603a40-71b1-470e-bc4c-424e7c0f418c kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:55Z" name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8.18c1f496b3ee9707 namespace: kserve-ci-e2e-test resourceVersion: "26092" uid: 7fcecba5-5f27-485f-9d7b-fa3af8d750f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 63603a40-71b1-470e-bc4c-424e7c0f418c kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:55Z" name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8.18c1f496b464a285 namespace: kserve-ci-e2e-test resourceVersion: "26093" uid: eaa3c628-005e-488c-b1e9-df3633576f63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 63603a40-71b1-470e-bc4c-424e7c0f418c kind: Event lastTimestamp: "2026-07-13T21:00:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:00:11Z" name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8.18c1f496c56a8fcb namespace: kserve-ci-e2e-test resourceVersion: "30206" uid: 606fd692-ff26-4e75-a3e7-f93b9f7d6011 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T21:00:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 63603a40-71b1-470e-bc4c-424e7c0f418c kind: Event lastTimestamp: "2026-07-13T21:00:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T21:00:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:00:09Z" name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8.18c1f517c0b6ccbf namespace: kserve-ci-e2e-test resourceVersion: "30185" uid: d45c2f81-18c5-4b0c-9424-c3553d582225 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T21:00:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 63603a40-71b1-470e-bc4c-424e7c0f418c kind: Event lastTimestamp: "2026-07-13T21:00:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T21:00:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:00:09Z" name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8.18c1f517c0b7318c namespace: kserve-ci-e2e-test resourceVersion: "30188" uid: ed825a17-deab-46eb-a845-d9d8da5b1a8f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T21:00:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 63603a40-71b1-470e-bc4c-424e7c0f418c kind: Event lastTimestamp: "2026-07-13T21:00:11Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T21:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:00:11Z" name: error-404-isvc-492f6-predictor-598d856bcd-kxgf8.18c1f5183a0dc01a namespace: kserve-ci-e2e-test resourceVersion: "30203" uid: e9864b1b-a596-4359-9d6c-6cffd38dbdf1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-492f6-predictor-598d856bcd namespace: kserve-ci-e2e-test resourceVersion: "26047" uid: 717f2e4e-ca67-4a8b-84c0-2aaa83a9209e kind: Event lastTimestamp: "2026-07-13T20:50:54Z" message: 'Created pod: error-404-isvc-492f6-predictor-598d856bcd-kxgf8' metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:54Z" name: error-404-isvc-492f6-predictor-598d856bcd.18c1f4969164f05d namespace: kserve-ci-e2e-test resourceVersion: "26055" uid: 83a9cd31-5b43-44ac-b768-bcf8700a3e73 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-492f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: 49a89780-5b94-48b5-8a91-c52e36272fc9 kind: Event lastTimestamp: "2026-07-13T20:50:54Z" message: Scaled up replica set error-404-isvc-492f6-predictor-598d856bcd from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:54Z" name: error-404-isvc-492f6-predictor.18c1f4968fec0cea namespace: kserve-ci-e2e-test resourceVersion: "26048" uid: 53f73c0c-86e5-43b1-ac16-a248b2d708a4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:51:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-492f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 78282a32-aeb3-4f73-8dde-f67e9386baea kind: Event lastTimestamp: "2026-07-13T20:51:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:39Z" name: error-404-isvc-492f6-predictor.18c1f49a0e51d798 namespace: kserve-ci-e2e-test resourceVersion: "26491" uid: da59b26f-9243-4e96-b7fc-602c9715c2d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:51:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-492f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26046" uid: 78282a32-aeb3-4f73-8dde-f67e9386baea kind: Event lastTimestamp: "2026-07-13T20:51:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:39Z" name: error-404-isvc-492f6-predictor.18c1f49a0eb94292 namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: d4d74485-3289-46d8-875a-55dac666d007 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:51:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-492f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26225" uid: 78282a32-aeb3-4f73-8dde-f67e9386baea kind: Event lastTimestamp: "2026-07-13T20:52:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:52:24Z" name: error-404-isvc-492f6-predictor.18c1f49d8d619c77 namespace: kserve-ci-e2e-test resourceVersion: "26936" uid: 8a03fd88-e060-400f-a87a-25d16fc00b93 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:51:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-492f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26225" uid: 78282a32-aeb3-4f73-8dde-f67e9386baea kind: Event lastTimestamp: "2026-07-13T20:52:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:52:24Z" name: error-404-isvc-492f6-predictor.18c1f49d8dc65611 namespace: kserve-ci-e2e-test resourceVersion: "26937" uid: 1044b184-bd9e-4db1-a217-86203c2132d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-492f6 namespace: kserve-ci-e2e-test resourceVersion: "26030" uid: adc81b6c-f411-4b03-a8ee-f34495621349 kind: Event lastTimestamp: "2026-07-13T20:51:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:51:42Z" name: error-404-isvc-492f6.18c1f4968fee657d namespace: kserve-ci-e2e-test resourceVersion: "26533" uid: ea3909db-37fc-4ac2-ad1e-e20c2de49d57 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-492f6 namespace: kserve-ci-e2e-test resourceVersion: "26030" uid: adc81b6c-f411-4b03-a8ee-f34495621349 kind: Event lastTimestamp: "2026-07-13T20:50:54Z" message: 'Failed to update status for InferenceService "error-404-isvc-492f6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-492f6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:50:54Z" name: error-404-isvc-492f6.18c1f496903b3e4e namespace: kserve-ci-e2e-test resourceVersion: "26051" uid: d7725c92-8375-4521-8811-db429a51f7c2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-492f6 namespace: kserve-ci-e2e-test resourceVersion: "26030" uid: adc81b6c-f411-4b03-a8ee-f34495621349 kind: Event lastTimestamp: "2026-07-13T20:50:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-492f6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:50:54Z" name: error-404-isvc-492f6.18c1f496903b96c2 namespace: kserve-ci-e2e-test resourceVersion: "26052" uid: 316ed08b-a556-45ce-ae95-04d9fae46c1a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-492f6 namespace: kserve-ci-e2e-test resourceVersion: "26530" uid: adc81b6c-f411-4b03-a8ee-f34495621349 kind: Event lastTimestamp: "2026-07-13T20:51:41Z" message: InferenceService [error-404-isvc-492f6] is Ready metadata: creationTimestamp: "2026-07-13T20:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:51:41Z" name: error-404-isvc-492f6.18c1f4a185699836 namespace: kserve-ci-e2e-test resourceVersion: "26531" uid: 580f37f8-eb4d-4198-b6cf-a97ddaadad49 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:50:44.276623Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6 namespace: kserve-ci-e2e-test resourceVersion: "25771" uid: bc85ef38-2865-43c7-a334-96fc8736d535 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-575ad-predictor-7d77b767d4-xbql6 to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:50:44Z" name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6.18c1f49415fc7c03 namespace: kserve-ci-e2e-test resourceVersion: "25775" uid: 71173155-fcd5-4522-a73a-d722c20a4c1b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: bc85ef38-2865-43c7-a334-96fc8736d535 kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-575ad-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:44Z" name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6.18c1f4942025e35e namespace: kserve-ci-e2e-test resourceVersion: "25790" uid: feafec75-8c35-4090-b542-d5ec9db5689b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6 namespace: kserve-ci-e2e-test resourceVersion: "25781" uid: bc85ef38-2865-43c7-a334-96fc8736d535 kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:50:45Z" name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6.18c1f4945443f7fa namespace: kserve-ci-e2e-test resourceVersion: "25843" uid: d401a6c0-d0f2-49f7-9f9a-b638033f4bb7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: bc85ef38-2865-43c7-a334-96fc8736d535 kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6.18c1f49455813a81 namespace: kserve-ci-e2e-test resourceVersion: "25845" uid: 8dda2fd1-cd27-4499-a46e-6f218ecb0dee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: bc85ef38-2865-43c7-a334-96fc8736d535 kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6.18c1f49457a54dc1 namespace: kserve-ci-e2e-test resourceVersion: "25849" uid: de077a6e-8547-4b84-b248-25e0c504c468 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: bc85ef38-2865-43c7-a334-96fc8736d535 kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6.18c1f494586d5121 namespace: kserve-ci-e2e-test resourceVersion: "25850" uid: 3089c1bc-f582-4b1a-9bff-b4c134896df2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: bc85ef38-2865-43c7-a334-96fc8736d535 kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6.18c1f494587f8e82 namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: bc99f2d1-9afb-4629-a4ff-39b8683ae6cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: bc85ef38-2865-43c7-a334-96fc8736d535 kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6.18c1f4945b7a9288 namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: 910dadc5-5d92-4e93-9a58-d7a2eae7f7e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: bc85ef38-2865-43c7-a334-96fc8736d535 kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6.18c1f4945bef75f3 namespace: kserve-ci-e2e-test resourceVersion: "25853" uid: bc977026-ca7b-4393-8295-437d3c78ef39 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:50:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: bc85ef38-2865-43c7-a334-96fc8736d535 kind: Event lastTimestamp: "2026-07-13T20:51:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:51:22Z" name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6.18c1f494a9f36941 namespace: kserve-ci-e2e-test resourceVersion: "26342" uid: 902072ff-ae72-46b7-86df-6e7de4c3fb89 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T21:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: bc85ef38-2865-43c7-a334-96fc8736d535 kind: Event lastTimestamp: "2026-07-13T21:08:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T21:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:08:13Z" name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6.18c1f58873a96e30 namespace: kserve-ci-e2e-test resourceVersion: "33534" uid: 22def732-16dd-45fb-b844-83bacd551715 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T21:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: bc85ef38-2865-43c7-a334-96fc8736d535 kind: Event lastTimestamp: "2026-07-13T21:08:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T21:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:08:13Z" name: error-404-isvc-575ad-predictor-7d77b767d4-xbql6.18c1f58873aa97a7 namespace: kserve-ci-e2e-test resourceVersion: "33535" uid: 0e669f16-74e0-4c3d-b86c-44c715833732 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-575ad-predictor-7d77b767d4 namespace: kserve-ci-e2e-test resourceVersion: "25764" uid: 597676d9-520a-4c00-ac88-072d265146e7 kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: 'Created pod: error-404-isvc-575ad-predictor-7d77b767d4-xbql6' metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:44Z" name: error-404-isvc-575ad-predictor-7d77b767d4.18c1f4941574078f namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: d92ef13b-4fce-423a-8cd1-20953d60d0b5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-575ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "25762" uid: 07bd654d-3d47-4376-80f4-40751960beac kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: Scaled up replica set error-404-isvc-575ad-predictor-7d77b767d4 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:44Z" name: error-404-isvc-575ad-predictor.18c1f49413fd41b7 namespace: kserve-ci-e2e-test resourceVersion: "25765" uid: 02d0ff08-1f9a-4585-a225-b929636a206a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:50:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-575ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "25763" uid: 97a7331f-ccbf-4f6a-8a0c-fb1918601149 kind: Event lastTimestamp: "2026-07-13T20:51:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:14Z" name: error-404-isvc-575ad-predictor.18c1f497928137e1 namespace: kserve-ci-e2e-test resourceVersion: "26278" uid: 15fb1746-db46-4bfe-964e-41758dd0265a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:50:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-575ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "25763" uid: 97a7331f-ccbf-4f6a-8a0c-fb1918601149 kind: Event lastTimestamp: "2026-07-13T20:51:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:14Z" name: error-404-isvc-575ad-predictor.18c1f49792edee82 namespace: kserve-ci-e2e-test resourceVersion: "26279" uid: 3bb380f1-b30f-444a-a4c9-44cb9dc28e8f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:51:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-575ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: 97a7331f-ccbf-4f6a-8a0c-fb1918601149 kind: Event lastTimestamp: "2026-07-13T20:51:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:44Z" name: error-404-isvc-575ad-predictor.18c1f49e9042d599 namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: eb1c5db8-4b15-4f7e-9ffe-b4b8733f5261 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:51:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-575ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: 97a7331f-ccbf-4f6a-8a0c-fb1918601149 kind: Event lastTimestamp: "2026-07-13T20:51:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:44Z" name: error-404-isvc-575ad-predictor.18c1f49e908fcd6f namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: 816caf7d-cc32-4833-8bb7-da5893eeaa71 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-575ad namespace: kserve-ci-e2e-test resourceVersion: "25746" uid: 456dfb2a-55d0-4749-8e98-e09d0ee6d91c kind: Event lastTimestamp: "2026-07-13T20:51:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:51:32Z" name: error-404-isvc-575ad.18c1f4941400e106 namespace: kserve-ci-e2e-test resourceVersion: "26440" uid: 63d522c0-181c-4e45-bc23-6e0e1d26f165 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-575ad namespace: kserve-ci-e2e-test resourceVersion: "25746" uid: 456dfb2a-55d0-4749-8e98-e09d0ee6d91c kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: 'Failed to update status for InferenceService "error-404-isvc-575ad": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-575ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:50:44Z" name: error-404-isvc-575ad.18c1f4941474f928 namespace: kserve-ci-e2e-test resourceVersion: "25768" uid: a9f7271b-02ea-43ec-b1d6-b3099e1ae4d9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-575ad namespace: kserve-ci-e2e-test resourceVersion: "25746" uid: 456dfb2a-55d0-4749-8e98-e09d0ee6d91c kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-575ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:50:44Z" name: error-404-isvc-575ad.18c1f49414758c36 namespace: kserve-ci-e2e-test resourceVersion: "25769" uid: 9d3f0cf7-0453-486f-9227-71fcc5db5d7e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-575ad namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 456dfb2a-55d0-4749-8e98-e09d0ee6d91c kind: Event lastTimestamp: "2026-07-13T20:51:32Z" message: InferenceService [error-404-isvc-575ad] is Ready metadata: creationTimestamp: "2026-07-13T20:51:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:51:32Z" name: error-404-isvc-575ad.18c1f49f6537e64a namespace: kserve-ci-e2e-test resourceVersion: "26439" uid: 2a15cfcf-c3df-41a8-8b1c-428acc2ea65c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:41:29.452353Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 2abd4c1c-f51f-41fd-b3c8-f114bfe24b7d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d.18c1f412e7e1ef0c namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: 6a555766-2b1d-4115-ac29-9c7b8aa47a7a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d namespace: kserve-ci-e2e-test resourceVersion: "21230" uid: 2abd4c1c-f51f-41fd-b3c8-f114bfe24b7d kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d.18c1f41302a9ccf1 namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 9ebddfa8-3c70-4532-9104-00ea89f596c8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 2abd4c1c-f51f-41fd-b3c8-f114bfe24b7d kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d.18c1f4130430c24d namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 381234e7-19b5-473a-92d8-43ac4ca6fbc0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 2abd4c1c-f51f-41fd-b3c8-f114bfe24b7d kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d.18c1f41306ed27dc namespace: kserve-ci-e2e-test resourceVersion: "21285" uid: a169e58a-16b3-4c27-8fcd-66926db919ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 2abd4c1c-f51f-41fd-b3c8-f114bfe24b7d kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d.18c1f413075ece9c namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: bfdfc048-20d6-403d-94a3-0df9464452c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 2abd4c1c-f51f-41fd-b3c8-f114bfe24b7d kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d.18c1f413076952fc namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: a317fb2e-2ca0-4b65-b485-ec43bc12e1fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 2abd4c1c-f51f-41fd-b3c8-f114bfe24b7d kind: Event lastTimestamp: "2026-07-13T20:41:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:30Z" name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d.18c1f4130a5a4279 namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: cf01b853-b6ef-4319-ab26-96c4369b12e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 2abd4c1c-f51f-41fd-b3c8-f114bfe24b7d kind: Event lastTimestamp: "2026-07-13T20:41:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:30Z" name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d.18c1f4130aedff57 namespace: kserve-ci-e2e-test resourceVersion: "21290" uid: a008aa4c-eb5e-421c-8caf-ce64d757125e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T20:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 2abd4c1c-f51f-41fd-b3c8-f114bfe24b7d kind: Event lastTimestamp: "2026-07-13T20:50:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:46Z" name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d.18c1f4132462784c namespace: kserve-ci-e2e-test resourceVersion: "25870" uid: 345f77ee-e794-452c-8938-2712266d402e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 2abd4c1c-f51f-41fd-b3c8-f114bfe24b7d kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:44Z" name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d.18c1f4940d2701f6 namespace: kserve-ci-e2e-test resourceVersion: "25728" uid: 8cbbfebf-4a2d-450e-9186-26168f0c0cde reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 2abd4c1c-f51f-41fd-b3c8-f114bfe24b7d kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:44Z" name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d.18c1f4940d278f20 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: 51dd66d8-a7ff-41c2-b35a-ea998a26f496 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 2abd4c1c-f51f-41fd-b3c8-f114bfe24b7d kind: Event lastTimestamp: "2026-07-13T20:50:46Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:46Z" name: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d.18c1f49498dbaf92 namespace: kserve-ci-e2e-test resourceVersion: "25869" uid: a4be6e28-8b52-45fd-a19c-86e9cb11096a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5859e-predictor-55f99fd8b8 namespace: kserve-ci-e2e-test resourceVersion: "21214" uid: 4f7c7fcb-437e-45ba-8a54-eb61d7517c5d kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: 'Created pod: error-404-isvc-5859e-predictor-55f99fd8b8-5mg6d' metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-5859e-predictor-55f99fd8b8.18c1f412e752b4bd namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: 4f683392-7561-4f4b-b29a-05131301d41a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5859e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21212" uid: 50293c5f-cee9-492c-b54b-c06fdfda94f5 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Scaled up replica set error-404-isvc-5859e-predictor-55f99fd8b8 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-5859e-predictor.18c1f412e5162687 namespace: kserve-ci-e2e-test resourceVersion: "21215" uid: c99c1ad1-1e31-437c-bdd1-87ed3b0b77df reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:41:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5859e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21213" uid: 6312b0af-07ed-4a90-9fc7-3e2bb9d5e8ad kind: Event lastTimestamp: "2026-07-13T20:41:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:59Z" name: error-404-isvc-5859e-predictor.18c1f41663c20732 namespace: kserve-ci-e2e-test resourceVersion: "21553" uid: 8abe838a-48cc-47de-8412-286344b9a7b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:41:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5859e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21213" uid: 6312b0af-07ed-4a90-9fc7-3e2bb9d5e8ad kind: Event lastTimestamp: "2026-07-13T20:41:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:59Z" name: error-404-isvc-5859e-predictor.18c1f4166457b0ac namespace: kserve-ci-e2e-test resourceVersion: "21554" uid: fd5fab20-0af1-4da2-88ed-3d7e9b24815f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:42:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5859e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: 6312b0af-07ed-4a90-9fc7-3e2bb9d5e8ad kind: Event lastTimestamp: "2026-07-13T20:42:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:42:29Z" name: error-404-isvc-5859e-predictor.18c1f41d61f8f3cb namespace: kserve-ci-e2e-test resourceVersion: "21821" uid: a9aa8595-9f3a-4eb6-a075-dbd73fcd9a92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:42:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5859e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: 6312b0af-07ed-4a90-9fc7-3e2bb9d5e8ad kind: Event lastTimestamp: "2026-07-13T20:42:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:42:29Z" name: error-404-isvc-5859e-predictor.18c1f41d626fa6c8 namespace: kserve-ci-e2e-test resourceVersion: "21822" uid: c7762c63-d197-4c5a-8c26-8871bdd83cae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5859e namespace: kserve-ci-e2e-test resourceVersion: "21193" uid: 36c0f242-5b75-4139-b28b-e5f15ce0c76c kind: Event lastTimestamp: "2026-07-13T20:42:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:42:16Z" name: error-404-isvc-5859e.18c1f412e5188f0a namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: 963c9022-bd35-4622-a5ad-a924d0a8fbae reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5859e namespace: kserve-ci-e2e-test resourceVersion: "21193" uid: 36c0f242-5b75-4139-b28b-e5f15ce0c76c kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: 'Failed to update status for InferenceService "error-404-isvc-5859e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5859e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-5859e.18c1f412e5968112 namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: c40cadff-2f13-4694-96e0-880d7f110215 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5859e namespace: kserve-ci-e2e-test resourceVersion: "21193" uid: 36c0f242-5b75-4139-b28b-e5f15ce0c76c kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5859e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-5859e.18c1f412e596e0c9 namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 956e3b03-eef7-451f-b972-dea5f7dfcea0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:42:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5859e namespace: kserve-ci-e2e-test resourceVersion: "21699" uid: 36c0f242-5b75-4139-b28b-e5f15ce0c76c kind: Event lastTimestamp: "2026-07-13T20:42:16Z" message: InferenceService [error-404-isvc-5859e] is Ready metadata: creationTimestamp: "2026-07-13T20:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:42:16Z" name: error-404-isvc-5859e.18c1f41de187df91 namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: 4fa635b8-e8ff-42bd-857a-b1d3fb2dd1ed reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:30:04.418302Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-91263-predictor-5f479d8546-d2nfn namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 16e207be-60cb-4faa-95e9-f6929e0d536b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-91263-predictor-5f479d8546-d2nfn to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:30:04Z" name: error-404-isvc-91263-predictor-5f479d8546-d2nfn.18c1f37368abf1c1 namespace: kserve-ci-e2e-test resourceVersion: "15263" uid: d39a76ba-f32b-45f1-afb6-c03d402be2a0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-91263-predictor-5f479d8546-d2nfn namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 16e207be-60cb-4faa-95e9-f6929e0d536b kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-91263-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:04Z" name: error-404-isvc-91263-predictor-5f479d8546-d2nfn.18c1f3737546f8ed namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: f459e362-adbb-45fb-bafb-e0a321af779f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-91263-predictor-5f479d8546-d2nfn namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 16e207be-60cb-4faa-95e9-f6929e0d536b kind: Event lastTimestamp: "2026-07-13T20:30:05Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:30:05Z" name: error-404-isvc-91263-predictor-5f479d8546-d2nfn.18c1f373a60c6bbf namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: aedb8c24-9931-45fa-96ab-7305658a3fc5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91263-predictor-5f479d8546-d2nfn namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 16e207be-60cb-4faa-95e9-f6929e0d536b kind: Event lastTimestamp: "2026-07-13T20:30:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:05Z" name: error-404-isvc-91263-predictor-5f479d8546-d2nfn.18c1f373a7e2dfca namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: 0cd48bc0-a1d3-4425-9e56-1a335bf9e74b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91263-predictor-5f479d8546-d2nfn namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 16e207be-60cb-4faa-95e9-f6929e0d536b kind: Event lastTimestamp: "2026-07-13T20:30:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:05Z" name: error-404-isvc-91263-predictor-5f479d8546-d2nfn.18c1f373ab2dee7c namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: 0aa94505-f0da-41cc-9c01-f350b6b49f5b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91263-predictor-5f479d8546-d2nfn namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 16e207be-60cb-4faa-95e9-f6929e0d536b kind: Event lastTimestamp: "2026-07-13T20:30:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:05Z" name: error-404-isvc-91263-predictor-5f479d8546-d2nfn.18c1f373aba26591 namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: ef8de3bd-9645-4da8-bb68-43609afa6240 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91263-predictor-5f479d8546-d2nfn namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 16e207be-60cb-4faa-95e9-f6929e0d536b kind: Event lastTimestamp: "2026-07-13T20:30:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:05Z" name: error-404-isvc-91263-predictor-5f479d8546-d2nfn.18c1f373abad29aa namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: 7e9f9969-9d0b-4264-9a5f-2507687be811 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91263-predictor-5f479d8546-d2nfn namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 16e207be-60cb-4faa-95e9-f6929e0d536b kind: Event lastTimestamp: "2026-07-13T20:30:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:05Z" name: error-404-isvc-91263-predictor-5f479d8546-d2nfn.18c1f373ae8bbdd6 namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 43c1dbb1-4078-471f-93f7-3371d5186e44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91263-predictor-5f479d8546-d2nfn namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 16e207be-60cb-4faa-95e9-f6929e0d536b kind: Event lastTimestamp: "2026-07-13T20:30:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:05Z" name: error-404-isvc-91263-predictor-5f479d8546-d2nfn.18c1f373af02673a namespace: kserve-ci-e2e-test resourceVersion: "15307" uid: 76d6f0d6-2547-4757-9ce6-1fd8e8f9965c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T20:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91263-predictor-5f479d8546-d2nfn namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 16e207be-60cb-4faa-95e9-f6929e0d536b kind: Event lastTimestamp: "2026-07-13T20:39:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:21Z" name: error-404-isvc-91263-predictor-5f479d8546-d2nfn.18c1f373b17331b2 namespace: kserve-ci-e2e-test resourceVersion: "19713" uid: 2ab54242-2fbb-4253-804e-104859a40090 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91263-predictor-5f479d8546-d2nfn namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 16e207be-60cb-4faa-95e9-f6929e0d536b kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:19Z" name: error-404-isvc-91263-predictor-5f479d8546-d2nfn.18c1f3f4bc491546 namespace: kserve-ci-e2e-test resourceVersion: "19676" uid: 7446bce1-a66d-4639-b8fb-76100575fef1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91263-predictor-5f479d8546-d2nfn namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 16e207be-60cb-4faa-95e9-f6929e0d536b kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:19Z" name: error-404-isvc-91263-predictor-5f479d8546-d2nfn.18c1f3f4bc4a31d2 namespace: kserve-ci-e2e-test resourceVersion: "19677" uid: 2b530288-5b2e-46d5-93ef-46f7e0a8e60c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91263-predictor-5f479d8546-d2nfn namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 16e207be-60cb-4faa-95e9-f6929e0d536b kind: Event lastTimestamp: "2026-07-13T20:39:21Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:21Z" name: error-404-isvc-91263-predictor-5f479d8546-d2nfn.18c1f3f525f9f9c9 namespace: kserve-ci-e2e-test resourceVersion: "19712" uid: 9b09c09f-f213-4c70-b6a5-335c1ed3404e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-91263-predictor-5f479d8546 namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: c3920218-7140-4cc1-a29c-41237b574b2f kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: 'Created pod: error-404-isvc-91263-predictor-5f479d8546-d2nfn' metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:04Z" name: error-404-isvc-91263-predictor-5f479d8546.18c1f373681c3306 namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: 2c5f92c3-9d41-4998-8afe-19a47ea9be0b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-91263-predictor namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: f493d883-b4dc-4bfa-a6b0-e3a71a45d6ce kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: Scaled up replica set error-404-isvc-91263-predictor-5f479d8546 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:04Z" name: error-404-isvc-91263-predictor.18c1f373659b4a86 namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: dbb816b3-8b7f-4e5d-bcbc-bf76e6aff44d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-91263-predictor namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: cc67d1b1-6696-4b34-981e-693be088a2af kind: Event lastTimestamp: "2026-07-13T20:30:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:19Z" name: error-404-isvc-91263-predictor.18c1f376e4128d98 namespace: kserve-ci-e2e-test resourceVersion: "15441" uid: 204626c4-d9ac-4aec-bce6-249f665cb06b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-91263-predictor namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: cc67d1b1-6696-4b34-981e-693be088a2af kind: Event lastTimestamp: "2026-07-13T20:30:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:19Z" name: error-404-isvc-91263-predictor.18c1f376e47213be namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: 36b8b695-3106-4983-a572-95b591a50153 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:30:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-91263-predictor namespace: kserve-ci-e2e-test resourceVersion: "15442" uid: cc67d1b1-6696-4b34-981e-693be088a2af kind: Event lastTimestamp: "2026-07-13T20:31:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:19Z" name: error-404-isvc-91263-predictor.18c1f37a633b662e namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: d96402b1-88aa-44b1-b014-1d6b5128a77e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:30:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-91263-predictor namespace: kserve-ci-e2e-test resourceVersion: "15442" uid: cc67d1b1-6696-4b34-981e-693be088a2af kind: Event lastTimestamp: "2026-07-13T20:31:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:19Z" name: error-404-isvc-91263-predictor.18c1f37a63b49966 namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: 100e16f1-c862-4ac3-b4f5-301e8f507f85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-91263 namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: 36021b70-dd8a-4646-b1f1-8e9d5b842598 kind: Event lastTimestamp: "2026-07-13T20:30:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:51Z" name: error-404-isvc-91263.18c1f37365a3fab6 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: 4365415c-298d-452d-924b-400678e9e785 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-91263 namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: 36021b70-dd8a-4646-b1f1-8e9d5b842598 kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-91263": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-91263": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:04Z" name: error-404-isvc-91263.18c1f3736600b10e namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: caf1366e-ff8c-4f95-a121-3ce1a2a867e4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-91263 namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: 36021b70-dd8a-4646-b1f1-8e9d5b842598 kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-91263": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:04Z" name: error-404-isvc-91263.18c1f373660114f7 namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 0df7fbfb-7d67-4c58-8531-223e57d2728a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-91263 namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 36021b70-dd8a-4646-b1f1-8e9d5b842598 kind: Event lastTimestamp: "2026-07-13T20:30:51Z" message: InferenceService [error-404-isvc-91263] is Ready metadata: creationTimestamp: "2026-07-13T20:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:51Z" name: error-404-isvc-91263.18c1f37e6d93f6e2 namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: f8ef5d09-0da6-4276-ac76-0fbab7152f71 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:28:34.207696Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14200" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:28:34Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f35e67b34a9a namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: 4875cb85-0013-47e0-97a9-9131ef5eb12b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-cbd18-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:34Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f35e74f56e94 namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: 428139d2-823d-418a-9b06-4988047c4206 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:28:35Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:28:35Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f35ea917caf1 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: dc7098fa-edde-4e8a-9d59-37f68b800eb8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:28:35Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1733" metadata: creationTimestamp: "2026-07-13T20:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:35Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f35eab0276d5 namespace: kserve-ci-e2e-test resourceVersion: "14307" uid: 8028fb72-d73c-441a-956b-20419860167f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:28:50Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1733" in 14.864s (14.864s including waiting). Image size: 1334693997 bytes.' metadata: creationTimestamp: "2026-07-13T20:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:50Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f36220fa996b namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: 8125cb62-e4e8-4afd-b385-9e703ff65531 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:28:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:50Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f36224f2b61c namespace: kserve-ci-e2e-test resourceVersion: "14463" uid: a86ebff1-0743-4f24-bdd3-2f8ce0ed198f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:28:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:50Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f3622590a1d5 namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: 901f003e-9b39-4737-aeef-114bdf437df3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:28:50Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-13T20:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:50Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f362259c95ad namespace: kserve-ci-e2e-test resourceVersion: "14467" uid: 897af1b8-c786-44f6-8dbf-4e35b86745aa reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:28:52Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.651s (2.651s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-13T20:28:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:52Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f362c3a1ad3d namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: 9fd6d1fe-f3df-43d1-826e-9997018c6550 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:28:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:53Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f362c7cd0401 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: f1fe5f6f-c750-4c55-b12b-09cb565a8a85 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:28:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:53Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f362c84e6e6f namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 6e6a5b10-915e-4fa2-b603-8dea3adae98e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:28:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:29:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:30Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f36319830321 namespace: kserve-ci-e2e-test resourceVersion: "14825" uid: 494e2b98-b294-486b-8c19-0a5ceaff749d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:04Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f373589d6661 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: d926aaf0-dc70-45c8-bf72-48de3ca60488 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:04Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f373589e232f namespace: kserve-ci-e2e-test resourceVersion: "15191" uid: 3b7f5e76-1d77-44a7-bdf8-1e2dc35468b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88 namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: d9c637a5-3e0d-49a6-8d80-aa5e33c8173e kind: Event lastTimestamp: "2026-07-13T20:30:05Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:05Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88.18c1f373a1a3d6e9 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 2ade53f4-beb7-4911-bf03-7074cb6010aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-cbd18-predictor-69b7b99dcb namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 10b59972-6518-4f6f-8d55-94a328f680e1 kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: 'Created pod: error-404-isvc-cbd18-predictor-69b7b99dcb-lxn88' metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:28:34Z" name: error-404-isvc-cbd18-predictor-69b7b99dcb.18c1f35e670e1583 namespace: kserve-ci-e2e-test resourceVersion: "14201" uid: bd2558eb-16f7-44d3-a94f-6742d40ce632 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-cbd18-predictor namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: 344097d0-642a-4420-bf82-6978ec03dfbc kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: Scaled up replica set error-404-isvc-cbd18-predictor-69b7b99dcb from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:28:34Z" name: error-404-isvc-cbd18-predictor.18c1f35e6539d7d0 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: d881f32d-92a3-42b9-8f18-25d80d1c2784 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:28:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cbd18-predictor namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: ace81f46-a0e9-4e2a-9e1d-7b518071226e kind: Event lastTimestamp: "2026-07-13T20:29:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:28:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:04Z" name: error-404-isvc-cbd18-predictor.18c1f361e3e07191 namespace: kserve-ci-e2e-test resourceVersion: "14583" uid: 06d68125-2689-4927-8c5f-29f2132c238b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:28:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cbd18-predictor namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: ace81f46-a0e9-4e2a-9e1d-7b518071226e kind: Event lastTimestamp: "2026-07-13T20:29:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:28:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:04Z" name: error-404-isvc-cbd18-predictor.18c1f361e451634b namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: f0148ab1-fee4-400d-bdfc-65bd86fcbeae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:29:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cbd18-predictor namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: ace81f46-a0e9-4e2a-9e1d-7b518071226e kind: Event lastTimestamp: "2026-07-13T20:29:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:49Z" name: error-404-isvc-cbd18-predictor.18c1f368e1e74132 namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: 08a07ceb-b775-4b79-a741-3c42fad90215 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:29:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cbd18-predictor namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: ace81f46-a0e9-4e2a-9e1d-7b518071226e kind: Event lastTimestamp: "2026-07-13T20:29:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:49Z" name: error-404-isvc-cbd18-predictor.18c1f368e2665534 namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: 5d1a689d-1191-4164-913c-f03273198f5f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cbd18 namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: 5dfe7bb1-3e5e-40c0-bd36-b19238196bff kind: Event lastTimestamp: "2026-07-13T20:29:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:29:40Z" name: error-404-isvc-cbd18.18c1f35e65410088 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: d1fb0493-070c-48dd-b736-bc927e986ed0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cbd18 namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: 5dfe7bb1-3e5e-40c0-bd36-b19238196bff kind: Event lastTimestamp: "2026-07-13T20:29:40Z" message: InferenceService [error-404-isvc-cbd18] is Ready metadata: creationTimestamp: "2026-07-13T20:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:29:40Z" name: error-404-isvc-cbd18.18c1f36dd5057180 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 0012ad47-e818-4016-a9b7-8eb160c0a5b1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:40:29.757173Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-db27e-predictor-649d5848b4-qx64q namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 2523fd26-9a63-4029-a430-73d2aa224288 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-db27e-predictor-649d5848b4-qx64q to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:40:29Z" name: error-404-isvc-db27e-predictor-649d5848b4-qx64q.18c1f40501c5d345 namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: b0ef08a9-9665-4e5d-8add-88c5ab79a010 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-db27e-predictor-649d5848b4-qx64q namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 2523fd26-9a63-4029-a430-73d2aa224288 kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-db27e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:29Z" name: error-404-isvc-db27e-predictor-649d5848b4-qx64q.18c1f4050f92fc80 namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 2284cb03-d942-4ab8-b02a-b357a7509b2c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-db27e-predictor-649d5848b4-qx64q namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: 2523fd26-9a63-4029-a430-73d2aa224288 kind: Event lastTimestamp: "2026-07-13T20:40:30Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:40:30Z" name: error-404-isvc-db27e-predictor-649d5848b4-qx64q.18c1f4054073ea18 namespace: kserve-ci-e2e-test resourceVersion: "20563" uid: 79d15cee-68da-49bb-90d7-3a147fa68440 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db27e-predictor-649d5848b4-qx64q namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 2523fd26-9a63-4029-a430-73d2aa224288 kind: Event lastTimestamp: "2026-07-13T20:40:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:30Z" name: error-404-isvc-db27e-predictor-649d5848b4-qx64q.18c1f405431397cf namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: e9d7ae58-0b95-43ae-920c-e86d182d5f96 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db27e-predictor-649d5848b4-qx64q namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 2523fd26-9a63-4029-a430-73d2aa224288 kind: Event lastTimestamp: "2026-07-13T20:40:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:30Z" name: error-404-isvc-db27e-predictor-649d5848b4-qx64q.18c1f4054719b52a namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 593571d4-be0a-49f9-bd41-57a6dfd7e96e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db27e-predictor-649d5848b4-qx64q namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 2523fd26-9a63-4029-a430-73d2aa224288 kind: Event lastTimestamp: "2026-07-13T20:40:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:30Z" name: error-404-isvc-db27e-predictor-649d5848b4-qx64q.18c1f405478f589e namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: 989663b6-09ca-43ae-a161-1892f4430b22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db27e-predictor-649d5848b4-qx64q namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 2523fd26-9a63-4029-a430-73d2aa224288 kind: Event lastTimestamp: "2026-07-13T20:40:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:30Z" name: error-404-isvc-db27e-predictor-649d5848b4-qx64q.18c1f4054798e570 namespace: kserve-ci-e2e-test resourceVersion: "20571" uid: 978a4396-7e02-4e56-92f0-4b1e3875fba6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db27e-predictor-649d5848b4-qx64q namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 2523fd26-9a63-4029-a430-73d2aa224288 kind: Event lastTimestamp: "2026-07-13T20:40:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:30Z" name: error-404-isvc-db27e-predictor-649d5848b4-qx64q.18c1f4054b99b87b namespace: kserve-ci-e2e-test resourceVersion: "20573" uid: e55cebb4-f6b9-42d2-9687-9da61d41c807 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db27e-predictor-649d5848b4-qx64q namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 2523fd26-9a63-4029-a430-73d2aa224288 kind: Event lastTimestamp: "2026-07-13T20:40:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:31Z" name: error-404-isvc-db27e-predictor-649d5848b4-qx64q.18c1f4054c2abc96 namespace: kserve-ci-e2e-test resourceVersion: "20574" uid: 0e2f5216-40f5-4173-a4eb-39f1519d68bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T20:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db27e-predictor-649d5848b4-qx64q namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 2523fd26-9a63-4029-a430-73d2aa224288 kind: Event lastTimestamp: "2026-07-13T20:49:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:47Z" name: error-404-isvc-db27e-predictor-649d5848b4-qx64q.18c1f40557d3df90 namespace: kserve-ci-e2e-test resourceVersion: "24938" uid: f806c92a-84bb-4626-9b9b-014d695b239f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db27e-predictor-649d5848b4-qx64q namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 2523fd26-9a63-4029-a430-73d2aa224288 kind: Event lastTimestamp: "2026-07-13T20:49:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:44Z" name: error-404-isvc-db27e-predictor-649d5848b4-qx64q.18c1f48634b9e037 namespace: kserve-ci-e2e-test resourceVersion: "24868" uid: 1f8ddfc2-88d4-4a39-ad57-9b16f97ed983 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db27e-predictor-649d5848b4-qx64q namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 2523fd26-9a63-4029-a430-73d2aa224288 kind: Event lastTimestamp: "2026-07-13T20:49:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:44Z" name: error-404-isvc-db27e-predictor-649d5848b4-qx64q.18c1f48634bb2a1a namespace: kserve-ci-e2e-test resourceVersion: "24872" uid: ab0c76ef-4953-4adc-8d2b-5b53d12afd48 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db27e-predictor-649d5848b4-qx64q namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 2523fd26-9a63-4029-a430-73d2aa224288 kind: Event lastTimestamp: "2026-07-13T20:49:47Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:47Z" name: error-404-isvc-db27e-predictor-649d5848b4-qx64q.18c1f486cc4ec6d9 namespace: kserve-ci-e2e-test resourceVersion: "24936" uid: 909456f0-9ffe-430f-abf2-e38ef6c6fc89 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-db27e-predictor-649d5848b4 namespace: kserve-ci-e2e-test resourceVersion: "20513" uid: c8b1d4e6-bcd3-40ea-9a08-7144269adf39 kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: 'Created pod: error-404-isvc-db27e-predictor-649d5848b4-qx64q' metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:29Z" name: error-404-isvc-db27e-predictor-649d5848b4.18c1f40501107672 namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 22983e46-a2a0-4fc5-a2f4-519bda10ebfd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-db27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: bea5d3b9-401b-4f93-b2d8-66b245ce4eea kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: Scaled up replica set error-404-isvc-db27e-predictor-649d5848b4 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:29Z" name: error-404-isvc-db27e-predictor.18c1f404fe7c65a1 namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: fafaf6ad-cfed-4bee-af05-ce07933b9212 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:40:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20512" uid: a3bba796-2e37-46cc-84b7-672695fa164d kind: Event lastTimestamp: "2026-07-13T20:40:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:59Z" name: error-404-isvc-db27e-predictor.18c1f4087d31c09d namespace: kserve-ci-e2e-test resourceVersion: "20832" uid: eda2e957-0cb1-4b83-8565-3421de7a7a7a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:40:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20512" uid: a3bba796-2e37-46cc-84b7-672695fa164d kind: Event lastTimestamp: "2026-07-13T20:40:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:59Z" name: error-404-isvc-db27e-predictor.18c1f4087dd9df93 namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: 82c19ea8-967e-4efc-91e2-8b84dd06ed1c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:41:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20705" uid: a3bba796-2e37-46cc-84b7-672695fa164d kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:41:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-db27e-predictor.18c1f40f7c21c4b2 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 1acef7cc-ba1f-4e61-adc7-fd1c15345db5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:41:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20705" uid: a3bba796-2e37-46cc-84b7-672695fa164d kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:41:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-db27e-predictor.18c1f40f7d156141 namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: 16627aa4-09d3-41ee-a0a1-0dec013fafd4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db27e namespace: kserve-ci-e2e-test resourceVersion: "20485" uid: c720f7de-40d4-47ef-9df8-5aa424abdd6c kind: Event lastTimestamp: "2026-07-13T20:41:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:17Z" name: error-404-isvc-db27e.18c1f404feb3886d namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: e8a7dfe1-deb9-44a6-919d-156c497aaf30 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db27e namespace: kserve-ci-e2e-test resourceVersion: "20485" uid: c720f7de-40d4-47ef-9df8-5aa424abdd6c kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: 'Failed to update status for InferenceService "error-404-isvc-db27e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-db27e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:40:29Z" name: error-404-isvc-db27e.18c1f404ff230574 namespace: kserve-ci-e2e-test resourceVersion: "20520" uid: ee27aa70-9e84-41fb-b00f-273ab91dcd8c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db27e namespace: kserve-ci-e2e-test resourceVersion: "20485" uid: c720f7de-40d4-47ef-9df8-5aa424abdd6c kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-db27e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:40:29Z" name: error-404-isvc-db27e.18c1f404ff234d3a namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 84284cda-2c7c-474b-ab3b-2829eaddaa83 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db27e namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: c720f7de-40d4-47ef-9df8-5aa424abdd6c kind: Event lastTimestamp: "2026-07-13T20:41:17Z" message: InferenceService [error-404-isvc-db27e] is Ready metadata: creationTimestamp: "2026-07-13T20:41:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:17Z" name: error-404-isvc-db27e.18c1f41016c91d0d namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 65625895-a295-46ac-8e82-683cd1a3e8f8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:40:19.159759Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5 namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: 8c486b63-5815-4c0d-880b-eaf4ba5c7844 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5 to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5.18c1f4028a1e14ff namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: 1d9d72b6-5d09-4354-9936-d8a68882a572 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5 namespace: kserve-ci-e2e-test resourceVersion: "20264" uid: 8c486b63-5815-4c0d-880b-eaf4ba5c7844 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5.18c1f402a4edebf8 namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: 3cf86c4b-9848-4227-915e-a4e7995726d8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5 namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 8c486b63-5815-4c0d-880b-eaf4ba5c7844 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5.18c1f402a6f27cf8 namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: 8a051059-29a5-4b07-821b-688771daf150 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5 namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 8c486b63-5815-4c0d-880b-eaf4ba5c7844 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5.18c1f402a971c1a1 namespace: kserve-ci-e2e-test resourceVersion: "20320" uid: 5dc9cff7-3812-45b6-85ef-cd191079e151 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5 namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 8c486b63-5815-4c0d-880b-eaf4ba5c7844 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5.18c1f402aa2ebae1 namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: fa599473-6676-4ebe-aff3-74a3fc3d50f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5 namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 8c486b63-5815-4c0d-880b-eaf4ba5c7844 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5.18c1f402aa39a112 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: f8bd583a-8465-460f-91aa-cff23829dc93 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5 namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 8c486b63-5815-4c0d-880b-eaf4ba5c7844 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5.18c1f402ad73fa2d namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: d09fed5f-d49e-4787-8b81-848fbd56e068 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5 namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 8c486b63-5815-4c0d-880b-eaf4ba5c7844 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5.18c1f402ade83688 namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: 1d2a9b42-0b7a-4f58-8e28-bd979c8b4d93 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5 namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 8c486b63-5815-4c0d-880b-eaf4ba5c7844 kind: Event lastTimestamp: "2026-07-13T20:40:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:56Z" name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5.18c1f402c4a8d4cb namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 825a8a4b-9eb7-4832-8062-08fe5da067ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5 namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 8c486b63-5815-4c0d-880b-eaf4ba5c7844 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5.18c1f412e0445118 namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: 4c719537-e807-405c-98fd-394504168445 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5 namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 8c486b63-5815-4c0d-880b-eaf4ba5c7844 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5.18c1f412e0463f3d namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: 7f28bdf4-4c7d-4e71-95f1-2b405ebf82c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5 namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 8c486b63-5815-4c0d-880b-eaf4ba5c7844 kind: Event lastTimestamp: "2026-07-13T20:41:31Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:31Z" name: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5.18c1f4134ce29ce1 namespace: kserve-ci-e2e-test resourceVersion: "21321" uid: 321de285-a059-4605-b644-5ac2432859cc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e8943-predictor-6c55cf5cbb namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: cc0503c3-d489-4b02-8f01-810d79d0adfd kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: 'Created pod: error-404-isvc-e8943-predictor-6c55cf5cbb-d4pj5' metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-e8943-predictor-6c55cf5cbb.18c1f402891b9be1 namespace: kserve-ci-e2e-test resourceVersion: "20256" uid: aba0c1ad-9abe-47f9-813d-9a7198e91c22 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e8943-predictor namespace: kserve-ci-e2e-test resourceVersion: "20234" uid: 15e1a561-a6c1-4163-9de7-a2e148182438 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Scaled up replica set error-404-isvc-e8943-predictor-6c55cf5cbb from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-e8943-predictor.18c1f402865938a1 namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: 9647b449-05f5-4916-96ad-dd998ca1b148 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:40:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e8943-predictor namespace: kserve-ci-e2e-test resourceVersion: "20241" uid: fb77a4ae-e937-4f58-b557-378a656ec2fd kind: Event lastTimestamp: "2026-07-13T20:40:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:49Z" name: error-404-isvc-e8943-predictor.18c1f40604f61a1f namespace: kserve-ci-e2e-test resourceVersion: "20747" uid: 1d602a13-a956-4cb3-9feb-0548e66f42d2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:40:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e8943-predictor namespace: kserve-ci-e2e-test resourceVersion: "20241" uid: fb77a4ae-e937-4f58-b557-378a656ec2fd kind: Event lastTimestamp: "2026-07-13T20:40:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:49Z" name: error-404-isvc-e8943-predictor.18c1f4060550f8f8 namespace: kserve-ci-e2e-test resourceVersion: "20748" uid: 662f4bb1-99d6-45aa-baad-687a28385bf2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:41:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e8943-predictor namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: fb77a4ae-e937-4f58-b557-378a656ec2fd kind: Event lastTimestamp: "2026-07-13T20:41:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:19Z" name: error-404-isvc-e8943-predictor.18c1f40d0336542e namespace: kserve-ci-e2e-test resourceVersion: "21044" uid: 570e0b73-f6c8-48f6-ab6c-28a117d8c75f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:41:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e8943-predictor namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: fb77a4ae-e937-4f58-b557-378a656ec2fd kind: Event lastTimestamp: "2026-07-13T20:41:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:19Z" name: error-404-isvc-e8943-predictor.18c1f40d03843b2a namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: 9111d2f5-2124-46f6-8f97-38917e66da9d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e8943 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: aeb78bc2-8de0-4bd9-b301-56d33ba79138 kind: Event lastTimestamp: "2026-07-13T20:41:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:06Z" name: error-404-isvc-e8943.18c1f402865c3a85 namespace: kserve-ci-e2e-test resourceVersion: "20900" uid: 77508c49-20e5-457d-a259-4599364aad45 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e8943 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: aeb78bc2-8de0-4bd9-b301-56d33ba79138 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: 'Failed to update status for InferenceService "error-404-isvc-e8943": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e8943": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-e8943.18c1f4028721064e namespace: kserve-ci-e2e-test resourceVersion: "20249" uid: 94661feb-9333-4591-8d5b-ec4275c0e64c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e8943 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: aeb78bc2-8de0-4bd9-b301-56d33ba79138 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e8943": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:40:19Z" name: error-404-isvc-e8943.18c1f40287215776 namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: be8f11e4-16ee-4872-ba6f-23ca756dedee reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e8943 namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: aeb78bc2-8de0-4bd9-b301-56d33ba79138 kind: Event lastTimestamp: "2026-07-13T20:41:06Z" message: InferenceService [error-404-isvc-e8943] is Ready metadata: creationTimestamp: "2026-07-13T20:41:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:06Z" name: error-404-isvc-e8943.18c1f40d81b3eb0a namespace: kserve-ci-e2e-test resourceVersion: "20896" uid: cb87d37b-42d1-4036-81bb-eddc490471b8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:28:33.921190Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd to ip-10-0-128-52.ec2.internal metadata: creationTimestamp: "2026-07-13T20:28:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:28:33Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f35e569f944b namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: 5d613648-7564-4103-9dfb-f908676169ed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14156" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:28:34Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f35e709df2de namespace: kserve-ci-e2e-test resourceVersion: "14229" uid: 6f5b0147-2542-4987-802b-cf8747e4b8d8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:10d11eb5432cd26f2fc3d3776649a96acdfb9765433f51db6bb597849bf71311" already present on machine metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:34Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f35e72542ab8 namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 8e9f7cca-e61e-4233-9536-3ed146bc4355 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:34Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f35e7598debe namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: db4f23da-8b74-4e56-8ab3-4f43ed3684cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:34Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f35e7611b07c namespace: kserve-ci-e2e-test resourceVersion: "14254" uid: 8cd4c980-e4a0-4608-b3ba-2595ad31d0c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:28:38Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1733" metadata: creationTimestamp: "2026-07-13T20:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:38Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f35f7f0d0663 namespace: kserve-ci-e2e-test resourceVersion: "14348" uid: 4dace443-726b-4f30-8c18-79ea99085870 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:29:07Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1733" in 28.177s (28.177s including waiting). Image size: 1555805621 bytes.' metadata: creationTimestamp: "2026-07-13T20:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:07Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f3660e8e524b namespace: kserve-ci-e2e-test resourceVersion: "14609" uid: 9cbb2b15-d12b-4f67-94dd-946cb46cd02e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:29:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:07Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f36612ee87f7 namespace: kserve-ci-e2e-test resourceVersion: "14611" uid: 93de7f85-0c48-42b7-9f09-e8754f102348 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:29:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:07Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f36613df5b73 namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: 124893fb-c073-43db-8a39-d2993c5e3c65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:29:07Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-13T20:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:07Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f36613ec310f namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 803e3326-0568-48ab-955c-42975af99faf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:29:09Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.832s (2.832s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-13T20:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:09Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f366bcbd91c6 namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: 269e6dd5-4d18-4fed-b24e-f820b337145d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:29:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:10Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f366c10e8413 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 3bc0be78-62a9-4c99-bffa-ffe0f8945f5c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:29:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:10Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f366c189c12f namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: 48f1af1e-7e28-48aa-9de2-f296af033542 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-13T20:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:31:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:06Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f366c5f05ed8 namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 92a52437-7b82-4849-a3f7-12e47733aa7a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:04Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f38150267833 namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: ee7b694d-2b65-436e-b2e6-0e56cd8d3b59 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:04Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f3815026e263 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: fa94523d-3085-416b-924f-e4c3dc410077 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 811dd1e2-0d2a-4aab-8a91-be98ac0f7946 kind: Event lastTimestamp: "2026-07-13T20:31:06Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:06Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd.18c1f381c856816b namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 0234dac0-cb4d-452e-ac39-dc93e139bcd6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6449ff9c59 namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: 98592f87-13dc-4e38-813c-b64462b73d1a kind: Event lastTimestamp: "2026-07-13T20:28:33Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6449ff9c59-zvxqd' metadata: creationTimestamp: "2026-07-13T20:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:28:33Z" name: isvc-sklearn-graph-1-predictor-6449ff9c59.18c1f35e5608d6f9 namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: f7ebb208-b715-417a-965c-4a42d6ac988c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: ef1be497-bdf0-44b9-bdc1-678d552a9260 kind: Event lastTimestamp: "2026-07-13T20:28:33Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6449ff9c59 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:28:33Z" name: isvc-sklearn-graph-1-predictor.18c1f35e4d6c770f namespace: kserve-ci-e2e-test resourceVersion: "14128" uid: 25213e5f-5578-404d-9cd8-1ea95488d83d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:28:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14137" uid: 67ad993c-8343-4a42-b853-c568643d49ec kind: Event lastTimestamp: "2026-07-13T20:29:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:48Z" name: isvc-sklearn-graph-1-predictor.18c1f361d301d8b5 namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 2a299863-ee9b-497f-b555-2d29b5f237f9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:28:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14137" uid: 67ad993c-8343-4a42-b853-c568643d49ec kind: Event lastTimestamp: "2026-07-13T20:29:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:48Z" name: isvc-sklearn-graph-1-predictor.18c1f361d3a9c655 namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: c4907027-3238-4036-8c9a-a18b200635c3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:29:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: 67ad993c-8343-4a42-b853-c568643d49ec kind: Event lastTimestamp: "2026-07-13T20:30:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:29:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:49Z" name: isvc-sklearn-graph-1-predictor.18c1f36c513de141 namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: a0545dca-7a8c-4d08-8613-c1e3d5ac6d98 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:29:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: 67ad993c-8343-4a42-b853-c568643d49ec kind: Event lastTimestamp: "2026-07-13T20:30:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:29:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:49Z" name: isvc-sklearn-graph-1-predictor.18c1f36c51a9a6f3 namespace: kserve-ci-e2e-test resourceVersion: "15696" uid: f7d215c5-a6ac-4145-b8e9-811fc1cc135f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-13T20:28:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14114" uid: d69e30f2-ff94-4b87-8be0-80ae993b2455 kind: Event lastTimestamp: "2026-07-13T20:30:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:26Z" name: isvc-sklearn-graph-1.18c1f35e537661bf namespace: kserve-ci-e2e-test resourceVersion: "15519" uid: ebc03f11-bc0c-4aa1-8bb4-afe2cf8f4ac6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14114" uid: d69e30f2-ff94-4b87-8be0-80ae993b2455 kind: Event lastTimestamp: "2026-07-13T20:28:33Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:28:33Z" name: isvc-sklearn-graph-1.18c1f35e53db2dfb namespace: kserve-ci-e2e-test resourceVersion: "14139" uid: a4641bbb-84fe-4bfb-b800-351de117537c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14114" uid: d69e30f2-ff94-4b87-8be0-80ae993b2455 kind: Event lastTimestamp: "2026-07-13T20:28:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:28:33Z" name: isvc-sklearn-graph-1.18c1f35e53db9bfc namespace: kserve-ci-e2e-test resourceVersion: "14140" uid: 5ee72975-e654-4024-9936-0bbfd78d10a1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: d69e30f2-ff94-4b87-8be0-80ae993b2455 kind: Event lastTimestamp: "2026-07-13T20:30:26Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-13T20:30:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:26Z" name: isvc-sklearn-graph-1.18c1f3787f7cc4db namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: f779411f-45f5-4273-b7fe-142092409039 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:28:34.515102Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv to ip-10-0-128-52.ec2.internal metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:28:34Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f35e7a05fa4b namespace: kserve-ci-e2e-test resourceVersion: "14262" uid: 785fc088-ce36-40c7-86f2-baa6bbcadc0a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:28:34Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f35e94a5bdbe namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 1ec3c450-63b5-419d-84a3-0e3bf57ce79a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:28:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:10d11eb5432cd26f2fc3d3776649a96acdfb9765433f51db6bb597849bf71311" already present on machine metadata: creationTimestamp: "2026-07-13T20:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:35Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f35e9a65f9f4 namespace: kserve-ci-e2e-test resourceVersion: "14296" uid: 4a3d05cf-0649-40c2-9d99-5ed6f8abd00d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:28:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T20:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:35Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f35e9dafde87 namespace: kserve-ci-e2e-test resourceVersion: "14297" uid: 5c180d93-bc94-42da-8b2b-c7db98a7bbf2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:28:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T20:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:35Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f35e9e2568b7 namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: 31130f8f-bc04-4860-b975-1a48955a8ede reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:28:38Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1733" metadata: creationTimestamp: "2026-07-13T20:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:38Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f35f7ef7f52c namespace: kserve-ci-e2e-test resourceVersion: "14347" uid: 2e75e79c-381f-49a3-9cb0-d0d17d83bdd2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:29:07Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1733" in 28.901s (28.901s including waiting). Image size: 1555805621 bytes.' metadata: creationTimestamp: "2026-07-13T20:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:07Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f36639a0c5c1 namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: eefb1064-2b93-4786-a614-a0c52b12e522 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:29:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:07Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f3663f7f9aa4 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: b8db7378-e1d3-4cad-8146-455ccd7a9aed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:29:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:07Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f36640c9b5cf namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: dba7399b-0435-41b6-8b31-26aee357878a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:29:07Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-13T20:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:07Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f36640eea24b namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: f497c0ab-7f8d-4436-a38f-54ec5fa866d9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:29:09Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.064s (2.064s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-13T20:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:09Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f366bbf7390d namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: 1b49933e-9cc8-4952-80f6-40b3bd404825 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:29:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:10Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f366c031be9f namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: 5e763481-2423-4076-8790-f21697ecfbb1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:29:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:10Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f366c0c17005 namespace: kserve-ci-e2e-test resourceVersion: "14648" uid: cce543df-657f-47e3-ada4-f15ec0aa1f70 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-13T20:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:31:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:06Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f366c5d6f69b namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: 32b15b0e-fc5f-43ec-9cc3-d5622a1a4ff3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:04Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f3814be0d5c8 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: e0c377e7-a303-44c6-b9e0-6a2eb051d332 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:04Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f3814be14b4c namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: 6954536a-e88a-47ed-b338-22c6c94fb22f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b60cc8a4-c2c4-46f5-87f9-073f3a8eee20 kind: Event lastTimestamp: "2026-07-13T20:31:06Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:06Z" name: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv.18c1f381c85600b9 namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 5b8ef0f2-c258-4d7d-aad3-1e3f727e5d0c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7894f545fc namespace: kserve-ci-e2e-test resourceVersion: "14244" uid: 7aed65e8-7a83-4d6c-b127-3866ee32e539 kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7894f545fc-xvkwv' metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:28:34Z" name: isvc-sklearn-graph-2-predictor-7894f545fc.18c1f35e7989fcac namespace: kserve-ci-e2e-test resourceVersion: "14259" uid: 14e6190e-7edf-4506-b6ab-ec3a335f2047 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14242" uid: fbadfd2f-bead-4f3d-994c-2ddac84f7c9c kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7894f545fc from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:28:34Z" name: isvc-sklearn-graph-2-predictor.18c1f35e7427970b namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 58fc5762-60a9-4b0e-b0f6-05baa2d15ac8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:28:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 7cdff31e-263c-443e-9a3f-21305fada188 kind: Event lastTimestamp: "2026-07-13T20:29:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:28:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:49Z" name: isvc-sklearn-graph-2-predictor.18c1f361f2a1a4d0 namespace: kserve-ci-e2e-test resourceVersion: "15007" uid: fa9da652-3f69-41c0-aa95-7df2900ad291 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:28:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 7cdff31e-263c-443e-9a3f-21305fada188 kind: Event lastTimestamp: "2026-07-13T20:29:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:28:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:49Z" name: isvc-sklearn-graph-2-predictor.18c1f361f3670c2d namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: 20181137-864a-4d70-b391-b1c70c9568c3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: 7cdff31e-263c-443e-9a3f-21305fada188 kind: Event lastTimestamp: "2026-07-13T20:30:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:49Z" name: isvc-sklearn-graph-2-predictor.18c1f3736db94614 namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: 3ea4aaf7-6321-434c-aec2-518a1321be0e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: 7cdff31e-263c-443e-9a3f-21305fada188 kind: Event lastTimestamp: "2026-07-13T20:30:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:49Z" name: isvc-sklearn-graph-2-predictor.18c1f3736e362c4a namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: 32d24691-e61c-49e8-9be3-2804913b0ff6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: 5d988200-1ec4-4da2-a9a1-9507c0f5c5d0 kind: Event lastTimestamp: "2026-07-13T20:30:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:26Z" name: isvc-sklearn-graph-2.18c1f35e744d815d namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 1072e4c6-2f35-4187-9832-ef5170113fad reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15516" uid: 5d988200-1ec4-4da2-a9a1-9507c0f5c5d0 kind: Event lastTimestamp: "2026-07-13T20:30:26Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-13T20:30:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:26Z" name: isvc-sklearn-graph-2.18c1f3788411748c namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: bc2d0ba1-fb84-4e23-902e-cf5189ec50da reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:28:34.360900Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-xcfvp to ip-10-0-128-52.ec2.internal metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:28:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f35e70d4ff2c namespace: kserve-ci-e2e-test resourceVersion: "14230" uid: 171b4f35-1dfa-4c7c-86a8-fb2298b841e7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:34Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f35e7cf38f51 namespace: kserve-ci-e2e-test resourceVersion: "14270" uid: 98512cc2-98dd-493c-a506-118bae491a4a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14239" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:28:35Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:28:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f35eaece7395 namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: 4e2c3ce7-e8d1-40f4-aaed-02f4c565667a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:28:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:10d11eb5432cd26f2fc3d3776649a96acdfb9765433f51db6bb597849bf71311" already present on machine metadata: creationTimestamp: "2026-07-13T20:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f35eb17958ed namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: e0a0c789-fa2d-4288-948c-331a20dc6cc9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:28:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T20:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f35eb57fea32 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 5397e3b7-214d-4155-9481-ac87766ad97a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:28:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T20:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f35eb5fa7236 namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: da383077-397c-48df-9f80-22bb8ef523b4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:28:39Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-13T20:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f35fbc0adc0b namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: e7b741ea-8ab5-4930-98d5-d4ee96707cb8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:29:07Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 27.471s (27.471s including waiting). Image size: 1305567991 bytes.' metadata: creationTimestamp: "2026-07-13T20:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f366217edf87 namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: ad6d42dc-85bb-4e45-8752-36e1682d5676 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:29:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f36626a365c1 namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: c8e1c23f-a69a-4d4a-86de-0d650b3e48de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:29:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f3662815cb61 namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: 7b5d82ae-b81e-4aa2-bb9d-c808a245683a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:29:07Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-13T20:29:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f36628223299 namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 8404f000-26df-4e20-873b-dae297c86aa6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:29:10Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.519s (2.519s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-13T20:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f366be4db0c6 namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: 1dc4dc5f-334d-4285-8416-9ab9803b7659 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:29:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f366c143704b namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 8a30f05e-a6be-486f-891a-9edd31b05ff2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:29:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f366c1bc0833 namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: b2b75daf-0a5e-4380-9448-10bd0a2bb1e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-13T20:29:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:31:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f36701e040b3 namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: d24c3b33-c771-43b0-932d-58ebf6510415 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f38148d8ac02 namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: 0437e355-9648-4860-846e-1e36a379275a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f38148d94490 namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 4ee91261-3651-46fa-9021-26506abb427c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: 98a6fde5-c82d-4a6d-a057-fdc3399a2025 kind: Event lastTimestamp: "2026-07-13T20:31:07Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-xcfvp.18c1f3820431d130 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 743583e7-8ccc-4a42-b80c-f6f6b0f9707b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-52.ec2.internal source: component: kubelet host: ip-10-0-128-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: d18d7ee4-e457-406a-8e8f-cb4d0d0dce86 kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-xcfvp' metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:28:34Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c1f35e701556d0 namespace: kserve-ci-e2e-test resourceVersion: "14228" uid: dbe8bc6a-bffc-4f4c-bf40-16263ab338f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14216" uid: 9d1c4d83-e5b6-4018-919c-625ebd2367fe kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:28:34Z" name: isvc-xgboost-graph-predictor.18c1f35e6cbe3b58 namespace: kserve-ci-e2e-test resourceVersion: "14219" uid: baa9ce5e-e0eb-4074-b219-89c98bc56947 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:28:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14217" uid: bd99f1b9-fdb0-4a3c-a200-ed51a4bce156 kind: Event lastTimestamp: "2026-07-13T20:29:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:28:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:34Z" name: isvc-xgboost-graph-predictor.18c1f361eb65e13e namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: eb7e7973-f85c-48da-9cf0-3514b82832a6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:28:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14217" uid: bd99f1b9-fdb0-4a3c-a200-ed51a4bce156 kind: Event lastTimestamp: "2026-07-13T20:29:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:28:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:34Z" name: isvc-xgboost-graph-predictor.18c1f361ebbe8c94 namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: e94771a9-8a35-4f33-be19-cb4575f14c84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:29:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: bd99f1b9-fdb0-4a3c-a200-ed51a4bce156 kind: Event lastTimestamp: "2026-07-13T20:30:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:34Z" name: isvc-xgboost-graph-predictor.18c1f36fe73169b8 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: ad49a8a3-1617-4cad-95ca-3e982bf3ebde reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:29:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: bd99f1b9-fdb0-4a3c-a200-ed51a4bce156 kind: Event lastTimestamp: "2026-07-13T20:30:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:34Z" name: isvc-xgboost-graph-predictor.18c1f36fe80b0556 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 9cf76828-a09f-4fdc-9999-13b85081a216 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14204" uid: 62d2ca95-b9d3-4dd3-b67b-062bbbaeadf0 kind: Event lastTimestamp: "2026-07-13T20:30:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:07Z" name: isvc-xgboost-graph.18c1f35e6cf4f8ce namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: b9dc0957-7c9d-4e3d-9dca-3f39e5cb61d7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: 62d2ca95-b9d3-4dd3-b67b-062bbbaeadf0 kind: Event lastTimestamp: "2026-07-13T20:30:07Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-13T20:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:07Z" name: isvc-xgboost-graph.18c1f3741227a5d8 namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: 4ac2662f-6c04-4583-ae37-5cc36c45ca09 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:30:53.827357Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-659b76d949-vdqbw namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: f93e1ab4-a893-4796-a2aa-e1b42bfaebde kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-659b76d949-vdqbw to ip-10-0-134-97.ec2.internal metadata: creationTimestamp: "2026-07-13T20:30:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:30:53Z" name: model-chainer-659b76d949-vdqbw.18c1f37ee9ae33cd namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: f1991c3c-bf52-4d5a-a98e-5ec363379f85 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:54Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-659b76d949-vdqbw namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: f93e1ab4-a893-4796-a2aa-e1b42bfaebde kind: Event lastTimestamp: "2026-07-13T20:30:54Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:30:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:30:54Z" name: model-chainer-659b76d949-vdqbw.18c1f37f030f7acb namespace: kserve-ci-e2e-test resourceVersion: "15786" uid: 55b15557-3beb-4626-8a64-ab07e65294a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-659b76d949-vdqbw namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: f93e1ab4-a893-4796-a2aa-e1b42bfaebde kind: Event lastTimestamp: "2026-07-13T20:30:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:8345cd1e0b8ad942d68e2fe65c99d692fac6452f8ea219efb6700aaf29b3e2fa" already present on machine metadata: creationTimestamp: "2026-07-13T20:30:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:54Z" name: model-chainer-659b76d949-vdqbw.18c1f37f04753232 namespace: kserve-ci-e2e-test resourceVersion: "15788" uid: 117238b7-66ca-4529-8cc7-4edd0eb01319 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-659b76d949-vdqbw namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: f93e1ab4-a893-4796-a2aa-e1b42bfaebde kind: Event lastTimestamp: "2026-07-13T20:30:54Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-13T20:30:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:54Z" name: model-chainer-659b76d949-vdqbw.18c1f37f079ec11e namespace: kserve-ci-e2e-test resourceVersion: "15789" uid: 40887f4f-6692-47ec-9144-4e9e30fa10aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-659b76d949-vdqbw namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: f93e1ab4-a893-4796-a2aa-e1b42bfaebde kind: Event lastTimestamp: "2026-07-13T20:30:54Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-13T20:30:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:54Z" name: model-chainer-659b76d949-vdqbw.18c1f37f0823f9c6 namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: 364700f9-1a28-469a-80b2-697c46da2938 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-659b76d949-vdqbw namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: f93e1ab4-a893-4796-a2aa-e1b42bfaebde kind: Event lastTimestamp: "2026-07-13T20:31:03Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-13T20:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:03Z" name: model-chainer-659b76d949-vdqbw.18c1f38143a920e2 namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: f9a342b4-a1e0-4345-9c68-aaf8d486a287 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-659b76d949-vdqbw namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: f93e1ab4-a893-4796-a2aa-e1b42bfaebde kind: Event lastTimestamp: "2026-07-13T20:31:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:30Z" name: model-chainer-659b76d949-vdqbw.18c1f381b7d1a4e0 namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 4cd5481b-fec9-466d-923e-0f2f4dfe80f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-659b76d949 namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: 78b84692-58aa-4c3f-9842-ba6bd413a7ba kind: Event lastTimestamp: "2026-07-13T20:30:53Z" message: 'Created pod: model-chainer-659b76d949-vdqbw' metadata: creationTimestamp: "2026-07-13T20:30:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:53Z" name: model-chainer-659b76d949.18c1f37ee8e33aec namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: 3d34f0ef-5aed-4c3c-be7b-16cc9c0952cc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 7bd7035f-42d5-4c9f-b527-521d6d118a6e kind: Event lastTimestamp: "2026-07-13T20:30:53Z" message: Scaled up replica set model-chainer-659b76d949 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:30:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:53Z" name: model-chainer.18c1f37ee7a292ad namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: 00d28461-8b5f-4910-8e9b-f6017cf43601 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: f8e67232-0850-4756-9064-83f57d2fbf2d kind: Event lastTimestamp: "2026-07-13T20:30:53Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-13T20:30:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:53Z" name: model-chainer.18c1f37eed4b184c namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: ef0c197f-9767-4466-afae-f6536791f19b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:32:04.198294Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-43619-cb57bcfb7-mjrs4 namespace: kserve-ci-e2e-test resourceVersion: "16566" uid: 4f3e95fb-3229-43dd-b1ae-1ac6888ba33e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-43619-cb57bcfb7-mjrs4 to ip-10-0-134-97.ec2.internal metadata: creationTimestamp: "2026-07-13T20:32:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:32:04Z" name: sequence-graph-43619-cb57bcfb7-mjrs4.18c1f38f4c1d780b namespace: kserve-ci-e2e-test resourceVersion: "16572" uid: eeb17efd-b4cc-45c9-9cff-c75578aa5be3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:32:04Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-43619-cb57bcfb7-mjrs4 namespace: kserve-ci-e2e-test resourceVersion: "16578" uid: 4f3e95fb-3229-43dd-b1ae-1ac6888ba33e kind: Event lastTimestamp: "2026-07-13T20:32:04Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:32:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:32:04Z" name: sequence-graph-43619-cb57bcfb7-mjrs4.18c1f38f65a8ed72 namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: 0183ec9f-ce2c-477f-81ae-59d15aa64365 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:32:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-43619} kind: Pod name: sequence-graph-43619-cb57bcfb7-mjrs4 namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: 4f3e95fb-3229-43dd-b1ae-1ac6888ba33e kind: Event lastTimestamp: "2026-07-13T20:32:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:8345cd1e0b8ad942d68e2fe65c99d692fac6452f8ea219efb6700aaf29b3e2fa" already present on machine metadata: creationTimestamp: "2026-07-13T20:32:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:32:04Z" name: sequence-graph-43619-cb57bcfb7-mjrs4.18c1f38f66fcb284 namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: 49989d78-4861-4976-b331-a2944fd75f73 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:32:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-43619} kind: Pod name: sequence-graph-43619-cb57bcfb7-mjrs4 namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: 4f3e95fb-3229-43dd-b1ae-1ac6888ba33e kind: Event lastTimestamp: "2026-07-13T20:32:04Z" message: 'Created container: sequence-graph-43619' metadata: creationTimestamp: "2026-07-13T20:32:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:32:04Z" name: sequence-graph-43619-cb57bcfb7-mjrs4.18c1f38f69ea2e83 namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: d214caf3-dbde-42c6-a016-83fc99045ad8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:32:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-43619} kind: Pod name: sequence-graph-43619-cb57bcfb7-mjrs4 namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: 4f3e95fb-3229-43dd-b1ae-1ac6888ba33e kind: Event lastTimestamp: "2026-07-13T20:32:04Z" message: Started container sequence-graph-43619 metadata: creationTimestamp: "2026-07-13T20:32:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:32:04Z" name: sequence-graph-43619-cb57bcfb7-mjrs4.18c1f38f6a7e4b0c namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: 1e8df575-cf2e-40c3-9e97-def3e385765a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-43619} kind: Pod name: sequence-graph-43619-cb57bcfb7-mjrs4 namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: 4f3e95fb-3229-43dd-b1ae-1ac6888ba33e kind: Event lastTimestamp: "2026-07-13T20:40:18Z" message: Stopping container sequence-graph-43619 metadata: creationTimestamp: "2026-07-13T20:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:18Z" name: sequence-graph-43619-cb57bcfb7-mjrs4.18c1f4027518073f namespace: kserve-ci-e2e-test resourceVersion: "20170" uid: 3bde4a00-cab4-4fb9-b6bf-a750a8a22e31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:40:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-43619} kind: Pod name: sequence-graph-43619-cb57bcfb7-mjrs4 namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: 4f3e95fb-3229-43dd-b1ae-1ac6888ba33e kind: Event lastTimestamp: "2026-07-13T20:40:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T20:40:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:46Z" name: sequence-graph-43619-cb57bcfb7-mjrs4.18c1f403018fd7fc namespace: kserve-ci-e2e-test resourceVersion: "20719" uid: 4c11f25d-6d64-4af6-a24e-81a7c121db92 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:32:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-43619-cb57bcfb7 namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: 05704af2-0844-4aac-a734-0ac2cb34e330 kind: Event lastTimestamp: "2026-07-13T20:32:04Z" message: 'Created pod: sequence-graph-43619-cb57bcfb7-mjrs4' metadata: creationTimestamp: "2026-07-13T20:32:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:32:04Z" name: sequence-graph-43619-cb57bcfb7.18c1f38f4b528079 namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: aa02536e-aa21-455e-ae30-9f0bd1fe1c10 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:32:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-43619 namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: 15993bb2-b472-4a19-82e8-2de24cf1dd77 kind: Event lastTimestamp: "2026-07-13T20:32:04Z" message: Scaled up replica set sequence-graph-43619-cb57bcfb7 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:32:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:32:04Z" name: sequence-graph-43619.18c1f38f4a22c840 namespace: kserve-ci-e2e-test resourceVersion: "16564" uid: 81611e50-7422-4598-a6e4-4ddee720ed80 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:32:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-43619 namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: 4b678129-9051-424f-80d9-ecba1410cebd kind: Event lastTimestamp: "2026-07-13T20:32:04Z" message: InferenceGraph [sequence-graph-43619] is Ready metadata: creationTimestamp: "2026-07-13T20:32:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:32:04Z" name: sequence-graph-43619.18c1f38f5145716b namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: fb965756-ee08-45f2-a20e-426de2668184 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:32:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-43619 namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 4b678129-9051-424f-80d9-ecba1410cebd kind: Event lastTimestamp: "2026-07-13T20:32:04Z" message: 'Failed to update status for InferenceGraph "sequence-graph-43619": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-43619": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:32:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:32:04Z" name: sequence-graph-43619.18c1f38f53f067ed namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 2cb1eded-dc6d-4050-8724-97daa79ff409 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:32:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-43619 namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 4b678129-9051-424f-80d9-ecba1410cebd kind: Event lastTimestamp: "2026-07-13T20:32:04Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-43619": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:32:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:32:04Z" name: sequence-graph-43619.18c1f38f53f0bb1f namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: 767f2bee-4ab8-418e-8f01-c8fb1d757938 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:32:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-43619 namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 655843ab-82ea-47e8-b44d-fbc0d5523112 kind: Event lastTimestamp: "2026-07-13T20:32:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:32:19Z" name: sequence-graph-43619.18c1f392c9042d67 namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: 3c01a22f-dfbb-460a-982d-2c184158da86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:32:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-43619 namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 655843ab-82ea-47e8-b44d-fbc0d5523112 kind: Event lastTimestamp: "2026-07-13T20:32:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:32:19Z" name: sequence-graph-43619.18c1f392c96d6b32 namespace: kserve-ci-e2e-test resourceVersion: "16711" uid: 7e4293ae-80e5-40a4-bf8b-d3e101d7dd0a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:32:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-43619 namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: 655843ab-82ea-47e8-b44d-fbc0d5523112 kind: Event lastTimestamp: "2026-07-13T20:32:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:32:34Z" name: sequence-graph-43619.18c1f396487596c7 namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: ef3b9c5f-685a-4c84-8d5b-9ffa731d0aea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:32:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-43619 namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: 655843ab-82ea-47e8-b44d-fbc0d5523112 kind: Event lastTimestamp: "2026-07-13T20:32:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:32:34Z" name: sequence-graph-43619.18c1f39648dc3d8e namespace: kserve-ci-e2e-test resourceVersion: "16813" uid: ad778880-d17b-4b9f-837f-9043a94edf94 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:42:29.387621Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-5859e-7c56564db7-krtf4 namespace: kserve-ci-e2e-test resourceVersion: "21803" uid: 85687986-8924-4a05-9a2e-e927954ed56c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-5859e-7c56564db7-krtf4 to ip-10-0-134-97.ec2.internal metadata: creationTimestamp: "2026-07-13T20:42:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:42:29Z" name: sequence-graph-5859e-7c56564db7-krtf4.18c1f420dc4d8b0b namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: 603cde3d-c00a-4932-b858-49b0e46ed773 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:42:29Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-5859e-7c56564db7-krtf4 namespace: kserve-ci-e2e-test resourceVersion: "21816" uid: 85687986-8924-4a05-9a2e-e927954ed56c kind: Event lastTimestamp: "2026-07-13T20:42:29Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:42:29Z" name: sequence-graph-5859e-7c56564db7-krtf4.18c1f420f5b93e61 namespace: kserve-ci-e2e-test resourceVersion: "21825" uid: f5113c46-5f92-4bd8-a969-0dcb160af32c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:42:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5859e} kind: Pod name: sequence-graph-5859e-7c56564db7-krtf4 namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: 85687986-8924-4a05-9a2e-e927954ed56c kind: Event lastTimestamp: "2026-07-13T20:42:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:8345cd1e0b8ad942d68e2fe65c99d692fac6452f8ea219efb6700aaf29b3e2fa" already present on machine metadata: creationTimestamp: "2026-07-13T20:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:42:29Z" name: sequence-graph-5859e-7c56564db7-krtf4.18c1f420f6e96daf namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: d62b31fe-6412-44a1-bf79-d0c5e3f53123 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:42:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5859e} kind: Pod name: sequence-graph-5859e-7c56564db7-krtf4 namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: 85687986-8924-4a05-9a2e-e927954ed56c kind: Event lastTimestamp: "2026-07-13T20:42:29Z" message: 'Created container: sequence-graph-5859e' metadata: creationTimestamp: "2026-07-13T20:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:42:29Z" name: sequence-graph-5859e-7c56564db7-krtf4.18c1f420f9cb5308 namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: 059a7839-fdf8-49b0-972f-0ae91b02d20f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:42:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5859e} kind: Pod name: sequence-graph-5859e-7c56564db7-krtf4 namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: 85687986-8924-4a05-9a2e-e927954ed56c kind: Event lastTimestamp: "2026-07-13T20:42:29Z" message: Started container sequence-graph-5859e metadata: creationTimestamp: "2026-07-13T20:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:42:29Z" name: sequence-graph-5859e-7c56564db7-krtf4.18c1f420fa3c0738 namespace: kserve-ci-e2e-test resourceVersion: "21829" uid: 921ca1d1-2bae-4c36-ba04-57f67af5ae92 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5859e} kind: Pod name: sequence-graph-5859e-7c56564db7-krtf4 namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: 85687986-8924-4a05-9a2e-e927954ed56c kind: Event lastTimestamp: "2026-07-13T20:50:43Z" message: Stopping container sequence-graph-5859e metadata: creationTimestamp: "2026-07-13T20:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:43Z" name: sequence-graph-5859e-7c56564db7-krtf4.18c1f49404c5a7e8 namespace: kserve-ci-e2e-test resourceVersion: "25685" uid: ab27a7f9-21a1-43f6-a34a-b452ebc4c483 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:50:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5859e} kind: Pod name: sequence-graph-5859e-7c56564db7-krtf4 namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: 85687986-8924-4a05-9a2e-e927954ed56c kind: Event lastTimestamp: "2026-07-13T20:51:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T20:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:51:11Z" name: sequence-graph-5859e-7c56564db7-krtf4.18c1f494a050a639 namespace: kserve-ci-e2e-test resourceVersion: "26247" uid: 349afe4c-edb8-4d82-93dc-b8b1ec155b7d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:42:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-5859e-7c56564db7 namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: cbb8da89-e448-4ba2-ac00-bb7c10b7be0f kind: Event lastTimestamp: "2026-07-13T20:42:29Z" message: 'Created pod: sequence-graph-5859e-7c56564db7-krtf4' metadata: creationTimestamp: "2026-07-13T20:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:42:29Z" name: sequence-graph-5859e-7c56564db7.18c1f420dbe679bd namespace: kserve-ci-e2e-test resourceVersion: "21805" uid: e5f78229-2a50-4df4-8e24-cfe59e632b5e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:42:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-5859e namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 542e3586-7d6d-4c8b-867a-9c17ebf20bec kind: Event lastTimestamp: "2026-07-13T20:42:29Z" message: Scaled up replica set sequence-graph-5859e-7c56564db7 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:42:29Z" name: sequence-graph-5859e.18c1f420db17a158 namespace: kserve-ci-e2e-test resourceVersion: "21801" uid: 89ee1866-588e-4dbf-9c7e-18f69e159367 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:42:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-5859e namespace: kserve-ci-e2e-test resourceVersion: "21818" uid: 1ba061ab-96b8-431f-9919-5d52e4d1e66d kind: Event lastTimestamp: "2026-07-13T20:42:29Z" message: InferenceGraph [sequence-graph-5859e] is Ready metadata: creationTimestamp: "2026-07-13T20:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:42:29Z" name: sequence-graph-5859e.18c1f420dfc5c255 namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: 9f79ac6f-d528-47c8-99a7-5ee7859274cc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:42:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-5859e namespace: kserve-ci-e2e-test resourceVersion: "21798" uid: a556f72e-db08-4e26-a7c9-188ad8d5aaf9 kind: Event lastTimestamp: "2026-07-13T20:42:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:42:44Z" name: sequence-graph-5859e.18c1f424598a301b namespace: kserve-ci-e2e-test resourceVersion: "21940" uid: b21ed93a-c095-4f26-859c-81de2119e515 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:42:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-5859e namespace: kserve-ci-e2e-test resourceVersion: "21798" uid: a556f72e-db08-4e26-a7c9-188ad8d5aaf9 kind: Event lastTimestamp: "2026-07-13T20:42:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:42:44Z" name: sequence-graph-5859e.18c1f42459ec1501 namespace: kserve-ci-e2e-test resourceVersion: "21942" uid: 102a51a2-a4ec-4754-bbc1-dc7e4913b5e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:42:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-5859e namespace: kserve-ci-e2e-test resourceVersion: "21941" uid: a556f72e-db08-4e26-a7c9-188ad8d5aaf9 kind: Event lastTimestamp: "2026-07-13T20:42:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:42:59Z" name: sequence-graph-5859e.18c1f427d90d01a8 namespace: kserve-ci-e2e-test resourceVersion: "22041" uid: 0fcfb760-1f4f-4d0b-9f4b-43dc533b7893 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:42:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-5859e namespace: kserve-ci-e2e-test resourceVersion: "21941" uid: a556f72e-db08-4e26-a7c9-188ad8d5aaf9 kind: Event lastTimestamp: "2026-07-13T20:42:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:42:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:42:59Z" name: sequence-graph-5859e.18c1f427d97827fc namespace: kserve-ci-e2e-test resourceVersion: "22044" uid: 9974c3ed-47c6-413f-9f7c-4422cd4dccfb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:41:19.064648Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e8943-86d8b689d5-jprd6 namespace: kserve-ci-e2e-test resourceVersion: "21026" uid: 76b7ee37-b200-4de2-8dbd-daeb4750066d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-e8943-86d8b689d5-jprd6 to ip-10-0-134-97.ec2.internal metadata: creationTimestamp: "2026-07-13T20:41:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:41:19Z" name: sequence-graph-e8943-86d8b689d5-jprd6.18c1f4107cba199f namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 9a6801ba-9545-4426-be37-3d751374c015 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:19Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e8943-86d8b689d5-jprd6 namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: 76b7ee37-b200-4de2-8dbd-daeb4750066d kind: Event lastTimestamp: "2026-07-13T20:41:19Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:41:19Z" name: sequence-graph-e8943-86d8b689d5-jprd6.18c1f410962677c5 namespace: kserve-ci-e2e-test resourceVersion: "21051" uid: b4b5902d-a3af-4f50-b9e8-da727f82e1bd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e8943} kind: Pod name: sequence-graph-e8943-86d8b689d5-jprd6 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 76b7ee37-b200-4de2-8dbd-daeb4750066d kind: Event lastTimestamp: "2026-07-13T20:41:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:8345cd1e0b8ad942d68e2fe65c99d692fac6452f8ea219efb6700aaf29b3e2fa" already present on machine metadata: creationTimestamp: "2026-07-13T20:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:19Z" name: sequence-graph-e8943-86d8b689d5-jprd6.18c1f410987e2281 namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: b816994c-1f46-4a1c-972f-b58ad6cbb75e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e8943} kind: Pod name: sequence-graph-e8943-86d8b689d5-jprd6 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 76b7ee37-b200-4de2-8dbd-daeb4750066d kind: Event lastTimestamp: "2026-07-13T20:41:19Z" message: 'Created container: sequence-graph-e8943' metadata: creationTimestamp: "2026-07-13T20:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:19Z" name: sequence-graph-e8943-86d8b689d5-jprd6.18c1f4109b8a599a namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 56d764ec-b8f8-491d-b3ed-c67bf9834c0b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e8943} kind: Pod name: sequence-graph-e8943-86d8b689d5-jprd6 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 76b7ee37-b200-4de2-8dbd-daeb4750066d kind: Event lastTimestamp: "2026-07-13T20:41:19Z" message: Started container sequence-graph-e8943 metadata: creationTimestamp: "2026-07-13T20:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:19Z" name: sequence-graph-e8943-86d8b689d5-jprd6.18c1f4109bfb9208 namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: 96a9a141-6b75-4955-a664-1cf1f3cfc9a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e8943} kind: Pod name: sequence-graph-e8943-86d8b689d5-jprd6 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 76b7ee37-b200-4de2-8dbd-daeb4750066d kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Stopping container sequence-graph-e8943 metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: sequence-graph-e8943-86d8b689d5-jprd6.18c1f412d3fe1f42 namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 809f39d9-c8d5-4bc4-b44f-e4db07bcc02c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:41:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e8943} kind: Pod name: sequence-graph-e8943-86d8b689d5-jprd6 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 76b7ee37-b200-4de2-8dbd-daeb4750066d kind: Event lastTimestamp: "2026-07-13T20:41:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T20:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:56Z" name: sequence-graph-e8943-86d8b689d5-jprd6.18c1f41356dd8237 namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: 1d749dbd-00a8-4a92-9196-0a845772bd39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-e8943-86d8b689d5 namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: 692beb87-ed07-434c-919e-6868c909b259 kind: Event lastTimestamp: "2026-07-13T20:41:19Z" message: 'Created pod: sequence-graph-e8943-86d8b689d5-jprd6' metadata: creationTimestamp: "2026-07-13T20:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:19Z" name: sequence-graph-e8943-86d8b689d5.18c1f4107bd26b66 namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: 8eebb540-a061-48d9-af25-69dc2a7ccc37 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-e8943 namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: bde1ef53-d985-4ab9-9d33-fb2d397b8afb kind: Event lastTimestamp: "2026-07-13T20:41:19Z" message: Scaled up replica set sequence-graph-e8943-86d8b689d5 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:19Z" name: sequence-graph-e8943.18c1f4107a5a2e09 namespace: kserve-ci-e2e-test resourceVersion: "21022" uid: d10b12c7-fc0a-4e91-821a-51ac2c8e7468 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e8943 namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 8695036e-bb5f-4784-adb9-5b8113e4ba84 kind: Event lastTimestamp: "2026-07-13T20:41:19Z" message: InferenceGraph [sequence-graph-e8943] is Ready metadata: creationTimestamp: "2026-07-13T20:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:19Z" name: sequence-graph-e8943.18c1f4108290d4ec namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: 9a8cafaf-628b-4bd4-beac-e54ee8e60099 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:50:44.586397Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-35019-8565bdbbb7-fcg64 namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: 6d137771-a26b-4357-b7a0-e4079ac333f8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-35019-8565bdbbb7-fcg64 to ip-10-0-134-97.ec2.internal metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:50:44Z" name: splitter-graph-35019-8565bdbbb7-fcg64.18c1f4942873390c namespace: kserve-ci-e2e-test resourceVersion: "25815" uid: 8f836359-d5e7-486b-9b82-d6980f972536 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-35019-8565bdbbb7-fcg64 namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: 6d137771-a26b-4357-b7a0-e4079ac333f8 kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:50:45Z" name: splitter-graph-35019-8565bdbbb7-fcg64.18c1f494424cb6fa namespace: kserve-ci-e2e-test resourceVersion: "25832" uid: 298f7cbb-5a7f-4934-899d-e548d53fd715 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-35019} kind: Pod name: splitter-graph-35019-8565bdbbb7-fcg64 namespace: kserve-ci-e2e-test resourceVersion: "25808" uid: 6d137771-a26b-4357-b7a0-e4079ac333f8 kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:8345cd1e0b8ad942d68e2fe65c99d692fac6452f8ea219efb6700aaf29b3e2fa" already present on machine metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: splitter-graph-35019-8565bdbbb7-fcg64.18c1f4944381350a namespace: kserve-ci-e2e-test resourceVersion: "25834" uid: e5fb8f2b-e902-4856-ad9d-7e1482cbde62 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-35019} kind: Pod name: splitter-graph-35019-8565bdbbb7-fcg64 namespace: kserve-ci-e2e-test resourceVersion: "25808" uid: 6d137771-a26b-4357-b7a0-e4079ac333f8 kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: 'Created container: splitter-graph-35019' metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: splitter-graph-35019-8565bdbbb7-fcg64.18c1f49446a12747 namespace: kserve-ci-e2e-test resourceVersion: "25835" uid: e923d932-ed8e-4bfb-9b62-f1529d828abb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-35019} kind: Pod name: splitter-graph-35019-8565bdbbb7-fcg64 namespace: kserve-ci-e2e-test resourceVersion: "25808" uid: 6d137771-a26b-4357-b7a0-e4079ac333f8 kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: Started container splitter-graph-35019 metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: splitter-graph-35019-8565bdbbb7-fcg64.18c1f49447171ace namespace: kserve-ci-e2e-test resourceVersion: "25836" uid: 0be21815-fe15-47fe-9b33-e7c00e35e424 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-35019} kind: Pod name: splitter-graph-35019-8565bdbbb7-fcg64 namespace: kserve-ci-e2e-test resourceVersion: "25808" uid: 6d137771-a26b-4357-b7a0-e4079ac333f8 kind: Event lastTimestamp: "2026-07-13T20:50:54Z" message: Stopping container splitter-graph-35019 metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:54Z" name: splitter-graph-35019-8565bdbbb7-fcg64.18c1f4968215041c namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 18466db6-00dc-4afc-9631-76c8f30ea2dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:50:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-35019} kind: Pod name: splitter-graph-35019-8565bdbbb7-fcg64 namespace: kserve-ci-e2e-test resourceVersion: "25808" uid: 6d137771-a26b-4357-b7a0-e4079ac333f8 kind: Event lastTimestamp: "2026-07-13T20:51:21Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T20:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:51:21Z" name: splitter-graph-35019-8565bdbbb7-fcg64.18c1f496f050f0d0 namespace: kserve-ci-e2e-test resourceVersion: "26330" uid: cd7136f5-5b8c-4b3d-aa97-0b37c3c36b27 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-35019-8565bdbbb7 namespace: kserve-ci-e2e-test resourceVersion: "25802" uid: 6a9dc040-370e-4006-80ec-a8be3e16ff7d kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: 'Created pod: splitter-graph-35019-8565bdbbb7-fcg64' metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:44Z" name: splitter-graph-35019-8565bdbbb7.18c1f49427ef3c25 namespace: kserve-ci-e2e-test resourceVersion: "25807" uid: 03f720de-9816-4955-b77f-959003aa201d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-35019 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: a1c2e9d9-09c4-4efc-8ac2-0f60c9928331 kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: Scaled up replica set splitter-graph-35019-8565bdbbb7 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:44Z" name: splitter-graph-35019.18c1f494275c2056 namespace: kserve-ci-e2e-test resourceVersion: "25803" uid: 6a9f0cfd-c4fe-4a13-bd86-8b9907ac83a8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-35019 namespace: kserve-ci-e2e-test resourceVersion: "25821" uid: 9ad6f889-3674-4879-99b4-4934dc8a16cf kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: InferenceGraph [splitter-graph-35019] is Ready metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:50:44Z" name: splitter-graph-35019.18c1f4942c0f38e0 namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: 94c1f4d5-ccdc-47b3-a76b-a0977b5b8673 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:51:54.912118Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-492f6-67c988957c-hcprv namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: 5a02f041-848b-4179-aaf7-b9291aaad1a9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-492f6-67c988957c-hcprv to ip-10-0-134-97.ec2.internal metadata: creationTimestamp: "2026-07-13T20:51:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:51:54Z" name: splitter-graph-492f6-67c988957c-hcprv.18c1f4a488309397 namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: e19e743b-af92-4e77-9980-7b73f2f2876f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:55Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-492f6-67c988957c-hcprv namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 5a02f041-848b-4179-aaf7-b9291aaad1a9 kind: Event lastTimestamp: "2026-07-13T20:51:55Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:51:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:51:55Z" name: splitter-graph-492f6-67c988957c-hcprv.18c1f4a4a22f3807 namespace: kserve-ci-e2e-test resourceVersion: "26695" uid: b536779f-5192-4097-8388-f675c4c166ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-492f6} kind: Pod name: splitter-graph-492f6-67c988957c-hcprv namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: 5a02f041-848b-4179-aaf7-b9291aaad1a9 kind: Event lastTimestamp: "2026-07-13T20:51:55Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:8345cd1e0b8ad942d68e2fe65c99d692fac6452f8ea219efb6700aaf29b3e2fa" already present on machine metadata: creationTimestamp: "2026-07-13T20:51:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:51:55Z" name: splitter-graph-492f6-67c988957c-hcprv.18c1f4a4a36dab6d namespace: kserve-ci-e2e-test resourceVersion: "26698" uid: cc7f9a2d-b4b5-46cd-b03b-342e42ed47b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-492f6} kind: Pod name: splitter-graph-492f6-67c988957c-hcprv namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: 5a02f041-848b-4179-aaf7-b9291aaad1a9 kind: Event lastTimestamp: "2026-07-13T20:51:55Z" message: 'Created container: splitter-graph-492f6' metadata: creationTimestamp: "2026-07-13T20:51:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:51:55Z" name: splitter-graph-492f6-67c988957c-hcprv.18c1f4a4a6463e8e namespace: kserve-ci-e2e-test resourceVersion: "26700" uid: b21d3c54-8a74-4c80-b95f-3ecd0b7e281d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-492f6} kind: Pod name: splitter-graph-492f6-67c988957c-hcprv namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: 5a02f041-848b-4179-aaf7-b9291aaad1a9 kind: Event lastTimestamp: "2026-07-13T20:51:55Z" message: Started container splitter-graph-492f6 metadata: creationTimestamp: "2026-07-13T20:51:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:51:55Z" name: splitter-graph-492f6-67c988957c-hcprv.18c1f4a4a6b5e650 namespace: kserve-ci-e2e-test resourceVersion: "26701" uid: 2e4f55f2-f155-4f33-afd3-04a327ff3a40 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T21:00:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-492f6} kind: Pod name: splitter-graph-492f6-67c988957c-hcprv namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: 5a02f041-848b-4179-aaf7-b9291aaad1a9 kind: Event lastTimestamp: "2026-07-13T21:00:09Z" message: Stopping container splitter-graph-492f6 metadata: creationTimestamp: "2026-07-13T21:00:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:00:09Z" name: splitter-graph-492f6-67c988957c-hcprv.18c1f517b53ab9b2 namespace: kserve-ci-e2e-test resourceVersion: "30154" uid: 97aba67f-edc6-4312-be15-8ab23cc9356b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T21:00:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-492f6} kind: Pod name: splitter-graph-492f6-67c988957c-hcprv namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: 5a02f041-848b-4179-aaf7-b9291aaad1a9 kind: Event lastTimestamp: "2026-07-13T21:00:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T21:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:00:36Z" name: splitter-graph-492f6-67c988957c-hcprv.18c1f51839ec05f4 namespace: kserve-ci-e2e-test resourceVersion: "30391" uid: 75495fba-0161-42e0-bd70-809174704983 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-492f6-67c988957c namespace: kserve-ci-e2e-test resourceVersion: "26664" uid: 9a5ddb76-3d34-4618-88ac-d10fcbe1d854 kind: Event lastTimestamp: "2026-07-13T20:51:54Z" message: 'Created pod: splitter-graph-492f6-67c988957c-hcprv' metadata: creationTimestamp: "2026-07-13T20:51:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:54Z" name: splitter-graph-492f6-67c988957c.18c1f4a487c8f6d0 namespace: kserve-ci-e2e-test resourceVersion: "26669" uid: 95b4c707-946a-4ed8-9c23-9501a0a2831e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-492f6 namespace: kserve-ci-e2e-test resourceVersion: "26661" uid: 9c50c9c6-da2a-4802-bd65-75b398ef919a kind: Event lastTimestamp: "2026-07-13T20:51:54Z" message: Scaled up replica set splitter-graph-492f6-67c988957c from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:51:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:54Z" name: splitter-graph-492f6.18c1f4a4872470e3 namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: f5b1615c-9f8a-400e-b375-a8c6e15d8921 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-492f6 namespace: kserve-ci-e2e-test resourceVersion: "26679" uid: f6f77104-2ea3-412a-b14d-28554ac49460 kind: Event lastTimestamp: "2026-07-13T20:51:54Z" message: InferenceGraph [splitter-graph-492f6] is Ready metadata: creationTimestamp: "2026-07-13T20:51:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:51:54Z" name: splitter-graph-492f6.18c1f4a48b811cc4 namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: cb364ce7-9042-4529-b179-95da004602cd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:52:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-492f6 namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: 70a40d45-59a7-4fce-963e-58b53686c64c kind: Event lastTimestamp: "2026-07-13T20:52:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:52:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:52:54Z" name: splitter-graph-492f6.18c1f4a8064eb5e8 namespace: kserve-ci-e2e-test resourceVersion: "27144" uid: 0cc1f05d-d31d-49e3-9c52-8a0c0a80b3c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:52:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-492f6 namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: 70a40d45-59a7-4fce-963e-58b53686c64c kind: Event lastTimestamp: "2026-07-13T20:52:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:52:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:52:54Z" name: splitter-graph-492f6.18c1f4a8070f92dd namespace: kserve-ci-e2e-test resourceVersion: "27145" uid: 1ae9599d-919f-417a-b99a-89989082b816 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:39:19.294180Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq namespace: kserve-ci-e2e-test resourceVersion: "19609" uid: 1e9d18b8-9dac-4e40-950d-488bc8b8c49f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq.18c1f3f499d9e17f namespace: kserve-ci-e2e-test resourceVersion: "19619" uid: f661d44e-bb2c-44aa-b5ca-a2dfc28538b2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq namespace: kserve-ci-e2e-test resourceVersion: "19627" uid: 1e9d18b8-9dac-4e40-950d-488bc8b8c49f kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq.18c1f3f4b3b03e7a namespace: kserve-ci-e2e-test resourceVersion: "19668" uid: d5298ad1-acf4-479c-bf3f-07701d48ca05 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 1e9d18b8-9dac-4e40-950d-488bc8b8c49f kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq.18c1f3f4b5785e09 namespace: kserve-ci-e2e-test resourceVersion: "19670" uid: 20c795c8-2467-4ea6-b335-dcda61c85161 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 1e9d18b8-9dac-4e40-950d-488bc8b8c49f kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq.18c1f3f4b8f1b88e namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: 289681b4-968a-4c24-8af0-b4fd845392ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 1e9d18b8-9dac-4e40-950d-488bc8b8c49f kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq.18c1f3f4b964d74b namespace: kserve-ci-e2e-test resourceVersion: "19674" uid: 4e97292f-cceb-4be6-a452-99476701911e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 1e9d18b8-9dac-4e40-950d-488bc8b8c49f kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq.18c1f3f4b97081ef namespace: kserve-ci-e2e-test resourceVersion: "19675" uid: 46052a94-eafa-4a3b-9c35-7cdf34078f6f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 1e9d18b8-9dac-4e40-950d-488bc8b8c49f kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq.18c1f3f4c278df2f namespace: kserve-ci-e2e-test resourceVersion: "19682" uid: 75e15eac-03e6-4cde-880b-adc244423751 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 1e9d18b8-9dac-4e40-950d-488bc8b8c49f kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq.18c1f3f4c2f554cc namespace: kserve-ci-e2e-test resourceVersion: "19683" uid: 65f331d5-6e22-46dc-9c0c-c73acc2ffc23 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:39:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 1e9d18b8-9dac-4e40-950d-488bc8b8c49f kind: Event lastTimestamp: "2026-07-13T20:39:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:56Z" name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq.18c1f3f4f8734932 namespace: kserve-ci-e2e-test resourceVersion: "19984" uid: 3184d5ec-5100-4f38-9908-a19df8216395 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 1e9d18b8-9dac-4e40-950d-488bc8b8c49f kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:29Z" name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq.18c1f404f727c166 namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: e07804cc-fc0d-4876-a9eb-496db53bb565 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 1e9d18b8-9dac-4e40-950d-488bc8b8c49f kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:29Z" name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq.18c1f404f7288ba9 namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: e13ca10a-cb3e-48b5-bb70-03d8dfda0db7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 1e9d18b8-9dac-4e40-950d-488bc8b8c49f kind: Event lastTimestamp: "2026-07-13T20:40:31Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:31Z" name: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq.18c1f40580beb543 namespace: kserve-ci-e2e-test resourceVersion: "20589" uid: 6e2dc059-fd38-4af1-b26a-0bbefc40adc5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-02ff4-predictor-6d45dbc5c7 namespace: kserve-ci-e2e-test resourceVersion: "19595" uid: 4abdfc21-b6b7-4450-8812-71fea20f81c0 kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: 'Created pod: success-200-isvc-02ff4-predictor-6d45dbc5c7-j66tq' metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-02ff4-predictor-6d45dbc5c7.18c1f3f4990f2e76 namespace: kserve-ci-e2e-test resourceVersion: "19611" uid: af3ae6ca-bf1c-485e-bd64-54c58d58c63d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-02ff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 9e498283-194a-4e41-9477-1e014e365af8 kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: Scaled up replica set success-200-isvc-02ff4-predictor-6d45dbc5c7 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-02ff4-predictor.18c1f3f4940bce9c namespace: kserve-ci-e2e-test resourceVersion: "19597" uid: b12106e2-55a8-4c38-bb8e-b1ca7708a59d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-02ff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19594" uid: 7cf30070-7a84-4009-82de-487822842e7e kind: Event lastTimestamp: "2026-07-13T20:39:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:39:34Z" name: success-200-isvc-02ff4-predictor.18c1f3f8133bb12b namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 32f589f9-a3ef-43f5-9ad7-bbbf53e72c82 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-02ff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19594" uid: 7cf30070-7a84-4009-82de-487822842e7e kind: Event lastTimestamp: "2026-07-13T20:39:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:39:34Z" name: success-200-isvc-02ff4-predictor.18c1f3f813e48c8d namespace: kserve-ci-e2e-test resourceVersion: "19816" uid: 8e553c08-8385-44c7-af60-e07c86e51f6d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:39:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-02ff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: 7cf30070-7a84-4009-82de-487822842e7e kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:19Z" name: success-200-isvc-02ff4-predictor.18c1f3fb92f881bc namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 131075e7-9f87-42d4-bcc7-9cae200d6dd3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:39:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-02ff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: 7cf30070-7a84-4009-82de-487822842e7e kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:19Z" name: success-200-isvc-02ff4-predictor.18c1f3fb934c49d9 namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: 8c0fc424-083d-40b7-8484-43355a8d7316 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-02ff4 namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 52aa43c2-a362-49b7-9706-3a1e482a2608 kind: Event lastTimestamp: "2026-07-13T20:40:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:40:07Z" name: success-200-isvc-02ff4.18c1f3f4943ea8e3 namespace: kserve-ci-e2e-test resourceVersion: "20069" uid: 165b4bb7-a009-4dc1-9723-cd347540f93b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-02ff4 namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 52aa43c2-a362-49b7-9706-3a1e482a2608 kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: 'Failed to update status for InferenceService "success-200-isvc-02ff4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-02ff4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-02ff4.18c1f3f494ed70c4 namespace: kserve-ci-e2e-test resourceVersion: "19603" uid: 98afab60-e2c7-4172-b6f7-e73f1b63894e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-02ff4 namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 52aa43c2-a362-49b7-9706-3a1e482a2608 kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-02ff4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-02ff4.18c1f3f494edc944 namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: 0ba769ec-5232-4c04-aa43-999ea0876364 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-02ff4 namespace: kserve-ci-e2e-test resourceVersion: "20066" uid: 52aa43c2-a362-49b7-9706-3a1e482a2608 kind: Event lastTimestamp: "2026-07-13T20:40:07Z" message: InferenceService [success-200-isvc-02ff4] is Ready metadata: creationTimestamp: "2026-07-13T20:40:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:40:07Z" name: success-200-isvc-02ff4.18c1f3ffb567c969 namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: 02402a56-385d-4c10-bbc9-76f86c499615 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:49:44.612766Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-35019-predictor-6647d995d5-4jxvp namespace: kserve-ci-e2e-test resourceVersion: "24841" uid: f722fc21-f418-4e36-8e1a-97dbc86f7d27 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-35019-predictor-6647d995d5-4jxvp to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:49:44Z" name: success-200-isvc-35019-predictor-6647d995d5-4jxvp.18c1f48631be4ff5 namespace: kserve-ci-e2e-test resourceVersion: "24859" uid: e3dfe884-9925-4dfd-b709-a0ea37b01228 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-35019-predictor-6647d995d5-4jxvp namespace: kserve-ci-e2e-test resourceVersion: "24870" uid: f722fc21-f418-4e36-8e1a-97dbc86f7d27 kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:49:45Z" name: success-200-isvc-35019-predictor-6647d995d5-4jxvp.18c1f4864bca8506 namespace: kserve-ci-e2e-test resourceVersion: "24899" uid: 66b80d28-8ca8-4ae8-801d-8e94b85cff6a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-35019-predictor-6647d995d5-4jxvp namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: f722fc21-f418-4e36-8e1a-97dbc86f7d27 kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:45Z" name: success-200-isvc-35019-predictor-6647d995d5-4jxvp.18c1f4864d3386e5 namespace: kserve-ci-e2e-test resourceVersion: "24901" uid: f57a4388-3b39-42fb-b3b4-9f526e8921b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-35019-predictor-6647d995d5-4jxvp namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: f722fc21-f418-4e36-8e1a-97dbc86f7d27 kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:45Z" name: success-200-isvc-35019-predictor-6647d995d5-4jxvp.18c1f48650e2144d namespace: kserve-ci-e2e-test resourceVersion: "24902" uid: dd6cf665-9d83-43bc-9ebf-22495bf76bf0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-35019-predictor-6647d995d5-4jxvp namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: f722fc21-f418-4e36-8e1a-97dbc86f7d27 kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:45Z" name: success-200-isvc-35019-predictor-6647d995d5-4jxvp.18c1f486515cc7d5 namespace: kserve-ci-e2e-test resourceVersion: "24903" uid: 886e11f4-75a9-4315-82aa-20a4f02008af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-35019-predictor-6647d995d5-4jxvp namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: f722fc21-f418-4e36-8e1a-97dbc86f7d27 kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:45Z" name: success-200-isvc-35019-predictor-6647d995d5-4jxvp.18c1f486516795cd namespace: kserve-ci-e2e-test resourceVersion: "24904" uid: 54cd1725-591c-4b3b-90fa-dee604ed1afe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-35019-predictor-6647d995d5-4jxvp namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: f722fc21-f418-4e36-8e1a-97dbc86f7d27 kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:45Z" name: success-200-isvc-35019-predictor-6647d995d5-4jxvp.18c1f48654cd627b namespace: kserve-ci-e2e-test resourceVersion: "24908" uid: 803c4411-22a5-4b71-8629-da16a5d4f3eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-35019-predictor-6647d995d5-4jxvp namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: f722fc21-f418-4e36-8e1a-97dbc86f7d27 kind: Event lastTimestamp: "2026-07-13T20:49:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:45Z" name: success-200-isvc-35019-predictor-6647d995d5-4jxvp.18c1f486554d3f50 namespace: kserve-ci-e2e-test resourceVersion: "24909" uid: 812c9ecf-65a8-42c4-92b7-c8baba26e2fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T20:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-35019-predictor-6647d995d5-4jxvp namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: f722fc21-f418-4e36-8e1a-97dbc86f7d27 kind: Event lastTimestamp: "2026-07-13T20:50:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:22Z" name: success-200-isvc-35019-predictor-6647d995d5-4jxvp.18c1f48666150da5 namespace: kserve-ci-e2e-test resourceVersion: "25428" uid: c559abbf-5c8e-4f60-b8e7-346b1fa75419 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-35019-predictor-6647d995d5-4jxvp namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: f722fc21-f418-4e36-8e1a-97dbc86f7d27 kind: Event lastTimestamp: "2026-07-13T20:50:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:54Z" name: success-200-isvc-35019-predictor-6647d995d5-4jxvp.18c1f49688010e22 namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: dbedb16a-8c7e-40a9-8342-d43d46a4dc34 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-35019-predictor-6647d995d5-4jxvp namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: f722fc21-f418-4e36-8e1a-97dbc86f7d27 kind: Event lastTimestamp: "2026-07-13T20:50:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:54Z" name: success-200-isvc-35019-predictor-6647d995d5-4jxvp.18c1f4968801c30f namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: 421ef19e-3d32-4155-81d5-3814f7201ad3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-35019-predictor-6647d995d5-4jxvp namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: f722fc21-f418-4e36-8e1a-97dbc86f7d27 kind: Event lastTimestamp: "2026-07-13T20:50:57Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:57Z" name: success-200-isvc-35019-predictor-6647d995d5-4jxvp.18c1f4972a1f9bd4 namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 7d948060-c66a-4460-836b-afd0cc45e427 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-35019-predictor-6647d995d5 namespace: kserve-ci-e2e-test resourceVersion: "24826" uid: 5dff12dc-7c45-4855-b625-1ffb96149f75 kind: Event lastTimestamp: "2026-07-13T20:49:44Z" message: 'Created pod: success-200-isvc-35019-predictor-6647d995d5-4jxvp' metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:49:44Z" name: success-200-isvc-35019-predictor-6647d995d5.18c1f48630b87857 namespace: kserve-ci-e2e-test resourceVersion: "24847" uid: b50330bf-56b6-4816-98cd-d80ab2083a77 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-35019-predictor namespace: kserve-ci-e2e-test resourceVersion: "24822" uid: 839cc924-834a-4c43-a21e-f8c67796da94 kind: Event lastTimestamp: "2026-07-13T20:49:44Z" message: Scaled up replica set success-200-isvc-35019-predictor-6647d995d5 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:49:44Z" name: success-200-isvc-35019-predictor.18c1f4862b5f9729 namespace: kserve-ci-e2e-test resourceVersion: "24829" uid: ee0e32b6-d209-4947-aabf-22d0b29401f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:49:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-35019-predictor namespace: kserve-ci-e2e-test resourceVersion: "24824" uid: 2c159a17-415d-402d-884b-2183f83540c7 kind: Event lastTimestamp: "2026-07-13T20:50:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:14Z" name: success-200-isvc-35019-predictor.18c1f489aa52bd30 namespace: kserve-ci-e2e-test resourceVersion: "25155" uid: 4415e6ea-fd51-4a87-8a64-17b319b49d47 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:49:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-35019-predictor namespace: kserve-ci-e2e-test resourceVersion: "24824" uid: 2c159a17-415d-402d-884b-2183f83540c7 kind: Event lastTimestamp: "2026-07-13T20:50:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:14Z" name: success-200-isvc-35019-predictor.18c1f489aab7fa48 namespace: kserve-ci-e2e-test resourceVersion: "25156" uid: 60bee447-a0fe-4bda-b619-891a6e3c858f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:50:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-35019-predictor namespace: kserve-ci-e2e-test resourceVersion: "25042" uid: 2c159a17-415d-402d-884b-2183f83540c7 kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:44Z" name: success-200-isvc-35019-predictor.18c1f490a8a1d04c namespace: kserve-ci-e2e-test resourceVersion: "25806" uid: 630299d5-333f-4ae5-a6a6-3a4b7125db96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:50:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-35019-predictor namespace: kserve-ci-e2e-test resourceVersion: "25042" uid: 2c159a17-415d-402d-884b-2183f83540c7 kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:44Z" name: success-200-isvc-35019-predictor.18c1f490a9057b22 namespace: kserve-ci-e2e-test resourceVersion: "25813" uid: b82fe28c-9a3d-4973-a1ec-ddf8de056159 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-35019 namespace: kserve-ci-e2e-test resourceVersion: "24802" uid: e1641457-bf57-4523-b322-e7abccb7ec4e kind: Event lastTimestamp: "2026-07-13T20:50:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:50:32Z" name: success-200-isvc-35019.18c1f4862b61c1be namespace: kserve-ci-e2e-test resourceVersion: "25596" uid: d5418718-d429-43c9-a006-69c70e02d74c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-35019 namespace: kserve-ci-e2e-test resourceVersion: "24802" uid: e1641457-bf57-4523-b322-e7abccb7ec4e kind: Event lastTimestamp: "2026-07-13T20:49:44Z" message: 'Failed to update status for InferenceService "success-200-isvc-35019": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-35019": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:49:44Z" name: success-200-isvc-35019.18c1f4862c0431ef namespace: kserve-ci-e2e-test resourceVersion: "24831" uid: eecfe8d2-7637-41c5-a257-ce057ac550ea reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-35019 namespace: kserve-ci-e2e-test resourceVersion: "24802" uid: e1641457-bf57-4523-b322-e7abccb7ec4e kind: Event lastTimestamp: "2026-07-13T20:49:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-35019": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:49:44Z" name: success-200-isvc-35019.18c1f4862c04b967 namespace: kserve-ci-e2e-test resourceVersion: "24832" uid: 4da252ec-96d6-44cf-88d1-d7ffdc20c8a9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-35019 namespace: kserve-ci-e2e-test resourceVersion: "25593" uid: e1641457-bf57-4523-b322-e7abccb7ec4e kind: Event lastTimestamp: "2026-07-13T20:50:32Z" message: InferenceService [success-200-isvc-35019] is Ready metadata: creationTimestamp: "2026-07-13T20:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:50:32Z" name: success-200-isvc-35019.18c1f4915e34643d namespace: kserve-ci-e2e-test resourceVersion: "25595" uid: 02febaae-7a0f-4e3a-9ca1-5cbfec3be511 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:31:04.166823Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-43619-predictor-857f7657f5-b28dc namespace: kserve-ci-e2e-test resourceVersion: "15950" uid: f18dc863-1999-46d0-aa30-37b4c9151890 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-43619-predictor-857f7657f5-b28dc to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:31:04Z" name: success-200-isvc-43619-predictor-857f7657f5-b28dc.18c1f38151f60310 namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: e271007c-87a5-4c23-a785-15ec8a9706b2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-43619-predictor-857f7657f5-b28dc namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: f18dc863-1999-46d0-aa30-37b4c9151890 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-43619-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:04Z" name: success-200-isvc-43619-predictor-857f7657f5-b28dc.18c1f3815da8373c namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: 8b647128-f083-42d3-8770-83de4acfba2e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-43619-predictor-857f7657f5-b28dc namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: f18dc863-1999-46d0-aa30-37b4c9151890 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:31:05Z" name: success-200-isvc-43619-predictor-857f7657f5-b28dc.18c1f3818faf35ae namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 2d78b2e8-8e4a-4d9c-b443-09afc6604569 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43619-predictor-857f7657f5-b28dc namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: f18dc863-1999-46d0-aa30-37b4c9151890 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:05Z" name: success-200-isvc-43619-predictor-857f7657f5-b28dc.18c1f38191844f55 namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: afa3e914-ffc8-4374-aa5b-aa5e37928905 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43619-predictor-857f7657f5-b28dc namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: f18dc863-1999-46d0-aa30-37b4c9151890 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:05Z" name: success-200-isvc-43619-predictor-857f7657f5-b28dc.18c1f38195f7dc35 namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: 938383c1-f846-4453-a16a-a9e73fa40ef3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43619-predictor-857f7657f5-b28dc namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: f18dc863-1999-46d0-aa30-37b4c9151890 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:05Z" name: success-200-isvc-43619-predictor-857f7657f5-b28dc.18c1f38196728bb7 namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: 5e71d3db-3510-4c1c-bbdf-1ec5a4a6b2f0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-43619-predictor-857f7657f5-b28dc namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: f18dc863-1999-46d0-aa30-37b4c9151890 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:05Z" name: success-200-isvc-43619-predictor-857f7657f5-b28dc.18c1f381967d94d7 namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: b738dd4e-7ee1-4a12-97ed-87974e2f5abf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-43619-predictor-857f7657f5-b28dc namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: f18dc863-1999-46d0-aa30-37b4c9151890 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:05Z" name: success-200-isvc-43619-predictor-857f7657f5-b28dc.18c1f38199a99392 namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 316beac4-88dc-4098-9dd5-01834ac25dba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-43619-predictor-857f7657f5-b28dc namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: f18dc863-1999-46d0-aa30-37b4c9151890 kind: Event lastTimestamp: "2026-07-13T20:31:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:05Z" name: success-200-isvc-43619-predictor-857f7657f5-b28dc.18c1f3819a392cc0 namespace: kserve-ci-e2e-test resourceVersion: "16059" uid: 3afb7f6f-15e1-42c5-af8b-4d9c0b757d87 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:31:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43619-predictor-857f7657f5-b28dc namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: f18dc863-1999-46d0-aa30-37b4c9151890 kind: Event lastTimestamp: "2026-07-13T20:31:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:42Z" name: success-200-isvc-43619-predictor-857f7657f5-b28dc.18c1f381f3e6e1ce namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: 0952b743-f79b-4c2d-b3f0-22535f2424e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43619-predictor-857f7657f5-b28dc namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: f18dc863-1999-46d0-aa30-37b4c9151890 kind: Event lastTimestamp: "2026-07-13T20:40:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:18Z" name: success-200-isvc-43619-predictor-857f7657f5-b28dc.18c1f4027ef1c5e1 namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: a25da2de-93ef-4c39-b1c5-83cc44e7113c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-43619-predictor-857f7657f5-b28dc namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: f18dc863-1999-46d0-aa30-37b4c9151890 kind: Event lastTimestamp: "2026-07-13T20:40:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:18Z" name: success-200-isvc-43619-predictor-857f7657f5-b28dc.18c1f4027ef238ec namespace: kserve-ci-e2e-test resourceVersion: "20207" uid: 71511a60-3d76-44d1-b8f9-38cb6804b1b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-43619-predictor-857f7657f5 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: b3175738-8d72-4c6c-a048-122d85de76dd kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: 'Created pod: success-200-isvc-43619-predictor-857f7657f5-b28dc' metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:04Z" name: success-200-isvc-43619-predictor-857f7657f5.18c1f381513b9853 namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 453a9bee-9364-4b2f-8f4b-1a6811750387 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-43619-predictor namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: f2d0b82b-7173-456f-9213-8025d5602a37 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: Scaled up replica set success-200-isvc-43619-predictor-857f7657f5 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:04Z" name: success-200-isvc-43619-predictor.18c1f3814f6d78eb namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: fdd6ffda-ea0c-49b6-9dee-127c7c461dbc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:31:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-43619-predictor namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: 823606c5-37f7-40c3-8994-f93525307f40 kind: Event lastTimestamp: "2026-07-13T20:31:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:31:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:34Z" name: success-200-isvc-43619-predictor.18c1f384ce2fee90 namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: 152a5bd2-cfea-4001-882c-800384011897 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:31:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-43619-predictor namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: 823606c5-37f7-40c3-8994-f93525307f40 kind: Event lastTimestamp: "2026-07-13T20:31:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:31:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:34Z" name: success-200-isvc-43619-predictor.18c1f384cec02ca2 namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 9315d665-ea78-4031-bf78-bea12020ad62 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:31:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-43619-predictor namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: 823606c5-37f7-40c3-8994-f93525307f40 kind: Event lastTimestamp: "2026-07-13T20:32:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:31:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:32:19Z" name: success-200-isvc-43619-predictor.18c1f38bccaf8cf0 namespace: kserve-ci-e2e-test resourceVersion: "16712" uid: 66434b4d-f85c-4c17-b8af-6f3c8bdf8cef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:31:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-43619-predictor namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: 823606c5-37f7-40c3-8994-f93525307f40 kind: Event lastTimestamp: "2026-07-13T20:32:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:31:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:32:19Z" name: success-200-isvc-43619-predictor.18c1f38bcd17f62c namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: f9d9ef55-7e3f-4178-8541-5a49fe06e069 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-43619 namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: 4a4fb6ca-3dbd-4ed0-a391-868cbee6eb75 kind: Event lastTimestamp: "2026-07-13T20:31:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:31:53Z" name: success-200-isvc-43619.18c1f3814f6e12a2 namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: ec631bd3-2ca2-420c-ab83-33dc7bfc4d36 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-43619 namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: 4a4fb6ca-3dbd-4ed0-a391-868cbee6eb75 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-43619": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-43619": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:31:04Z" name: success-200-isvc-43619.18c1f3814fd4b876 namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: a655446a-c3d3-4076-ba5c-aa0cc52a3467 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-43619 namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: 4a4fb6ca-3dbd-4ed0-a391-868cbee6eb75 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-43619": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:31:04Z" name: success-200-isvc-43619.18c1f3814fd519f9 namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: aca52f7a-0032-4015-bac2-bc170e6ba6ba reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-43619 namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: 4a4fb6ca-3dbd-4ed0-a391-868cbee6eb75 kind: Event lastTimestamp: "2026-07-13T20:31:52Z" message: InferenceService [success-200-isvc-43619] is Ready metadata: creationTimestamp: "2026-07-13T20:31:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:31:52Z" name: success-200-isvc-43619.18c1f38cb0595032 namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: 110ddf6e-9303-475d-803f-68f6a1e4825b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:50:54.851730Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw namespace: kserve-ci-e2e-test resourceVersion: "26023" uid: f5fc5723-4d05-4cea-ba54-22cea2393bec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:50:54Z" name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw.18c1f4968c4fd220 namespace: kserve-ci-e2e-test resourceVersion: "26031" uid: e8f01460-bd69-451a-bf76-ea321956cfe3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: f5fc5723-4d05-4cea-ba54-22cea2393bec kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:50:55Z" name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw.18c1f496a658daf9 namespace: kserve-ci-e2e-test resourceVersion: "26077" uid: dd3ab4f0-65fa-4ff9-947a-f5b3930004de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: f5fc5723-4d05-4cea-ba54-22cea2393bec kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:55Z" name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw.18c1f496a8104ee1 namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: 7e345c61-efbc-4018-b3a2-5894a69f2785 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: f5fc5723-4d05-4cea-ba54-22cea2393bec kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:55Z" name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw.18c1f496acc7531b namespace: kserve-ci-e2e-test resourceVersion: "26082" uid: fe698d36-b9b4-44de-8275-97db23b204f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: f5fc5723-4d05-4cea-ba54-22cea2393bec kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:55Z" name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw.18c1f496ad41dcaf namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 45d08c80-4ba1-44c3-a251-eb1ddd62b6ce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: f5fc5723-4d05-4cea-ba54-22cea2393bec kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:55Z" name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw.18c1f496ad4c54bc namespace: kserve-ci-e2e-test resourceVersion: "26085" uid: 53a2573e-730a-49b3-a998-f618220fe3ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: f5fc5723-4d05-4cea-ba54-22cea2393bec kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:55Z" name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw.18c1f496b0754c67 namespace: kserve-ci-e2e-test resourceVersion: "26087" uid: 85699fe2-bc17-47a0-8ce1-396b15ee2e13 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: f5fc5723-4d05-4cea-ba54-22cea2393bec kind: Event lastTimestamp: "2026-07-13T20:50:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:55Z" name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw.18c1f496b100b728 namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 4e62d60c-6d71-4e1b-b2b0-1f02767515cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T20:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: f5fc5723-4d05-4cea-ba54-22cea2393bec kind: Event lastTimestamp: "2026-07-13T21:00:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:00:11Z" name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw.18c1f496c57f44b9 namespace: kserve-ci-e2e-test resourceVersion: "30205" uid: ceb454e9-7b52-4841-8964-10346dbd5583 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T21:00:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: f5fc5723-4d05-4cea-ba54-22cea2393bec kind: Event lastTimestamp: "2026-07-13T21:00:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T21:00:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:00:09Z" name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw.18c1f517beb73767 namespace: kserve-ci-e2e-test resourceVersion: "30171" uid: 9218eaa4-5c94-4e28-98ca-886146a763a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T21:00:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: f5fc5723-4d05-4cea-ba54-22cea2393bec kind: Event lastTimestamp: "2026-07-13T21:00:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T21:00:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:00:09Z" name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw.18c1f517beb81a54 namespace: kserve-ci-e2e-test resourceVersion: "30172" uid: d3b72d87-6d4e-4e93-bd4e-355ed6c77710 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T21:00:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: f5fc5723-4d05-4cea-ba54-22cea2393bec kind: Event lastTimestamp: "2026-07-13T21:00:11Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T21:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:00:11Z" name: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw.18c1f5183a0ee63f namespace: kserve-ci-e2e-test resourceVersion: "30204" uid: 9315a74f-1b46-4e6e-95ae-721019e03684 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-492f6-predictor-6698d5fcc5 namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: d451b27f-244d-4640-8b6b-10c2c3098b78 kind: Event lastTimestamp: "2026-07-13T20:50:54Z" message: 'Created pod: success-200-isvc-492f6-predictor-6698d5fcc5-m8sgw' metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:54Z" name: success-200-isvc-492f6-predictor-6698d5fcc5.18c1f4968be0be2d namespace: kserve-ci-e2e-test resourceVersion: "26026" uid: b77a336a-fb3f-4930-a967-b7f79535b990 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-492f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26006" uid: 5f75c978-a56b-4f16-9b14-972f7b6c3933 kind: Event lastTimestamp: "2026-07-13T20:50:54Z" message: Scaled up replica set success-200-isvc-492f6-predictor-6698d5fcc5 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:54Z" name: success-200-isvc-492f6-predictor.18c1f4968a26572d namespace: kserve-ci-e2e-test resourceVersion: "26014" uid: fcba6670-f25b-4358-b219-d54b3a52f43f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-492f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26010" uid: ae6a2195-c314-4cbb-af24-fef160aec8cd kind: Event lastTimestamp: "2026-07-13T20:51:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:09Z" name: success-200-isvc-492f6-predictor.18c1f49a08ab6f1e namespace: kserve-ci-e2e-test resourceVersion: "26221" uid: 50d7ce53-e998-43cf-97c9-c7faf3dc221f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-492f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26010" uid: ae6a2195-c314-4cbb-af24-fef160aec8cd kind: Event lastTimestamp: "2026-07-13T20:51:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:09Z" name: success-200-isvc-492f6-predictor.18c1f49a090e87e1 namespace: kserve-ci-e2e-test resourceVersion: "26223" uid: 5e7a7820-50d1-4ef5-9be3-b4847578e2ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:51:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-492f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: ae6a2195-c314-4cbb-af24-fef160aec8cd kind: Event lastTimestamp: "2026-07-13T20:52:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:52:09Z" name: success-200-isvc-492f6-predictor.18c1f49d881b6008 namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: c65154f9-ea27-4c92-9469-3abc5ae1549e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:51:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-492f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: ae6a2195-c314-4cbb-af24-fef160aec8cd kind: Event lastTimestamp: "2026-07-13T20:52:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:52:09Z" name: success-200-isvc-492f6-predictor.18c1f49d88828b9d namespace: kserve-ci-e2e-test resourceVersion: "26825" uid: 3d448d6d-0f65-47b3-9d54-35cbf91ef34c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-492f6 namespace: kserve-ci-e2e-test resourceVersion: "25985" uid: 8e11de69-05d8-4b24-a655-95fe29f5cded kind: Event lastTimestamp: "2026-07-13T20:51:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:51:42Z" name: success-200-isvc-492f6.18c1f4968a29a8ed namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: 2224f52c-f304-485c-a0e5-ab284638ce7b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-492f6 namespace: kserve-ci-e2e-test resourceVersion: "25985" uid: 8e11de69-05d8-4b24-a655-95fe29f5cded kind: Event lastTimestamp: "2026-07-13T20:50:54Z" message: 'Failed to update status for InferenceService "success-200-isvc-492f6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-492f6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:50:54Z" name: success-200-isvc-492f6.18c1f4968af91ffd namespace: kserve-ci-e2e-test resourceVersion: "26021" uid: 2897909b-4717-4a28-bd4c-6de80fbd4aca reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-492f6 namespace: kserve-ci-e2e-test resourceVersion: "25985" uid: 8e11de69-05d8-4b24-a655-95fe29f5cded kind: Event lastTimestamp: "2026-07-13T20:50:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-492f6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:50:54Z" name: success-200-isvc-492f6.18c1f4968af96e96 namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: c328773a-f56f-42ff-9953-286c4d8aceba reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-492f6 namespace: kserve-ci-e2e-test resourceVersion: "26526" uid: 8e11de69-05d8-4b24-a655-95fe29f5cded kind: Event lastTimestamp: "2026-07-13T20:51:41Z" message: InferenceService [success-200-isvc-492f6] is Ready metadata: creationTimestamp: "2026-07-13T20:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:51:41Z" name: success-200-isvc-492f6.18c1f4a180ef5a3d namespace: kserve-ci-e2e-test resourceVersion: "26527" uid: 359df95c-2869-4c76-a117-1a3e9b7f53e3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:50:44.181048Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw namespace: kserve-ci-e2e-test resourceVersion: "25741" uid: 1bf0ceca-cfb7-49b1-9f7c-4e9bff37a15d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:50:44Z" name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw.18c1f494104a1c21 namespace: kserve-ci-e2e-test resourceVersion: "25745" uid: e5769add-156d-4d2d-bb38-fb753127559d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 1bf0ceca-cfb7-49b1-9f7c-4e9bff37a15d kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-575ad-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:44Z" name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw.18c1f4941a295211 namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: b4b48f44-c62a-41b5-aa21-c3513ac078d6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw namespace: kserve-ci-e2e-test resourceVersion: "25751" uid: 1bf0ceca-cfb7-49b1-9f7c-4e9bff37a15d kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:50:45Z" name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw.18c1f4944e5f3079 namespace: kserve-ci-e2e-test resourceVersion: "25837" uid: 1f7959ea-e71a-4977-acdf-bee7cfff606b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 1bf0ceca-cfb7-49b1-9f7c-4e9bff37a15d kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw.18c1f4944fd56461 namespace: kserve-ci-e2e-test resourceVersion: "25839" uid: 5d8b1290-d08b-41f9-93e3-db527bb871ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 1bf0ceca-cfb7-49b1-9f7c-4e9bff37a15d kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw.18c1f49453416a45 namespace: kserve-ci-e2e-test resourceVersion: "25840" uid: 6c536758-583a-4780-9fb2-b7d885935b67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 1bf0ceca-cfb7-49b1-9f7c-4e9bff37a15d kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw.18c1f49453c677ba namespace: kserve-ci-e2e-test resourceVersion: "25841" uid: cddf7429-2710-4c35-bca8-54e8da501052 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 1bf0ceca-cfb7-49b1-9f7c-4e9bff37a15d kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw.18c1f49453d2c33d namespace: kserve-ci-e2e-test resourceVersion: "25842" uid: 5b0e3944-02a0-434f-b942-18bc6b842bb6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 1bf0ceca-cfb7-49b1-9f7c-4e9bff37a15d kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw.18c1f494571705d3 namespace: kserve-ci-e2e-test resourceVersion: "25846" uid: 05b884bc-fbf6-4dec-8c50-6eef22e12bb4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 1bf0ceca-cfb7-49b1-9f7c-4e9bff37a15d kind: Event lastTimestamp: "2026-07-13T20:50:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:45Z" name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw.18c1f49457984705 namespace: kserve-ci-e2e-test resourceVersion: "25847" uid: 89af71c9-5acf-4e2a-8009-e9562caace88 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T20:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 1bf0ceca-cfb7-49b1-9f7c-4e9bff37a15d kind: Event lastTimestamp: "2026-07-13T20:51:31Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:51:31Z" name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw.18c1f4946e06126d namespace: kserve-ci-e2e-test resourceVersion: "26419" uid: 420546e4-3bf1-4bbb-8fb1-cba724f633ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T21:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 1bf0ceca-cfb7-49b1-9f7c-4e9bff37a15d kind: Event lastTimestamp: "2026-07-13T21:08:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T21:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:08:13Z" name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw.18c1f58870b9f8f9 namespace: kserve-ci-e2e-test resourceVersion: "33518" uid: ad17d386-7dfa-4d2a-b2f5-7c9df4c8c22f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T21:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 1bf0ceca-cfb7-49b1-9f7c-4e9bff37a15d kind: Event lastTimestamp: "2026-07-13T21:08:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T21:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:08:13Z" name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw.18c1f58870ba93a5 namespace: kserve-ci-e2e-test resourceVersion: "33519" uid: b1d50450-5802-4c9d-9138-8cff50b5a077 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T21:08:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 1bf0ceca-cfb7-49b1-9f7c-4e9bff37a15d kind: Event lastTimestamp: "2026-07-13T21:08:17Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-13T21:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:08:17Z" name: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw.18c1f5895e66420a namespace: kserve-ci-e2e-test resourceVersion: "33601" uid: 03adda44-32fe-406a-8a41-699373bb8c20 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-575ad-predictor-9f95b5ddf namespace: kserve-ci-e2e-test resourceVersion: "25731" uid: 21c42f20-441f-467d-bf4d-c5135715f090 kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: 'Created pod: success-200-isvc-575ad-predictor-9f95b5ddf-p6vcw' metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:44Z" name: success-200-isvc-575ad-predictor-9f95b5ddf.18c1f4940fdd8707 namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: 3d78edbb-7efb-48ef-9f3d-380b593518e8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-575ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "25727" uid: b15ff72a-7401-42f0-886d-723a70ea8d78 kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: Scaled up replica set success-200-isvc-575ad-predictor-9f95b5ddf from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:50:44Z" name: success-200-isvc-575ad-predictor.18c1f4940df16136 namespace: kserve-ci-e2e-test resourceVersion: "25734" uid: 4b8e0cf3-b82e-47d7-8df5-4bac33942037 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:50:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-575ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "25729" uid: 55596815-f1bd-4393-9252-264540d80286 kind: Event lastTimestamp: "2026-07-13T20:51:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:14Z" name: success-200-isvc-575ad-predictor.18c1f4978c3b9954 namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: 8ccf19a8-64c1-43a2-ae51-e6ac54cb9149 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:50:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-575ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "25729" uid: 55596815-f1bd-4393-9252-264540d80286 kind: Event lastTimestamp: "2026-07-13T20:51:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:14Z" name: success-200-isvc-575ad-predictor.18c1f4978ca0c031 namespace: kserve-ci-e2e-test resourceVersion: "26276" uid: e27c5f08-8fc4-4ffd-9eca-22ee16f4117d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:51:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-575ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "26137" uid: 55596815-f1bd-4393-9252-264540d80286 kind: Event lastTimestamp: "2026-07-13T20:51:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:59Z" name: success-200-isvc-575ad-predictor.18c1f49e89f36287 namespace: kserve-ci-e2e-test resourceVersion: "26730" uid: eac3a9e5-8e89-4b38-931b-8b13d2deebde reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:51:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-575ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "26137" uid: 55596815-f1bd-4393-9252-264540d80286 kind: Event lastTimestamp: "2026-07-13T20:51:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:59Z" name: success-200-isvc-575ad-predictor.18c1f49e8a594e5c namespace: kserve-ci-e2e-test resourceVersion: "26731" uid: 4f75e80c-10f8-4727-8d20-f371ff88da36 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-575ad namespace: kserve-ci-e2e-test resourceVersion: "25703" uid: 72259375-89d9-4654-ad4d-e1a37ca94a17 kind: Event lastTimestamp: "2026-07-13T20:51:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:51:41Z" name: success-200-isvc-575ad.18c1f4940dc54549 namespace: kserve-ci-e2e-test resourceVersion: "26528" uid: a014e731-50f7-4013-a2ed-8248e0a5dac6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-575ad namespace: kserve-ci-e2e-test resourceVersion: "25703" uid: 72259375-89d9-4654-ad4d-e1a37ca94a17 kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: 'Failed to update status for InferenceService "success-200-isvc-575ad": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-575ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:50:44Z" name: success-200-isvc-575ad.18c1f4940e24c421 namespace: kserve-ci-e2e-test resourceVersion: "25736" uid: 4e12cda7-4467-4711-92e1-90db28f30774 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-575ad namespace: kserve-ci-e2e-test resourceVersion: "25703" uid: 72259375-89d9-4654-ad4d-e1a37ca94a17 kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-575ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:50:44Z" name: success-200-isvc-575ad.18c1f4940e253531 namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 3873ef9b-d189-40f4-b0ea-cc20eb91e926 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-575ad namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 72259375-89d9-4654-ad4d-e1a37ca94a17 kind: Event lastTimestamp: "2026-07-13T20:51:41Z" message: InferenceService [success-200-isvc-575ad] is Ready metadata: creationTimestamp: "2026-07-13T20:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:51:41Z" name: success-200-isvc-575ad.18c1f4a17e45bf57 namespace: kserve-ci-e2e-test resourceVersion: "26522" uid: 89a37f9e-b4b7-48f1-83af-f71d8ff64192 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:41:29.346532Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 5b93aa5d-53e3-4993-ae82-3896cc5c6423 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5859e-predictor-76c758fc77-sq8sr to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr.18c1f412e193509b namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 84bf612d-5755-41a8-af3d-aa1fa5f8038e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 5b93aa5d-53e3-4993-ae82-3896cc5c6423 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr.18c1f412fc282c9e namespace: kserve-ci-e2e-test resourceVersion: "21268" uid: 84df6d89-dbaa-41dd-b1b0-775a3e4560fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 5b93aa5d-53e3-4993-ae82-3896cc5c6423 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr.18c1f412fdaf0db7 namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 03db3a61-d2d6-4480-ab51-3228304fd9d5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 5b93aa5d-53e3-4993-ae82-3896cc5c6423 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr.18c1f41301eb48a0 namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: f7425c5c-954c-45d8-bdeb-f46a51ca1689 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 5b93aa5d-53e3-4993-ae82-3896cc5c6423 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr.18c1f41302677389 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 8e2cd7b6-1687-4971-8935-af862fd7fe30 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 5b93aa5d-53e3-4993-ae82-3896cc5c6423 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr.18c1f4130272e846 namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: c1939252-692b-4644-82ed-f70a19fb9f9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 5b93aa5d-53e3-4993-ae82-3896cc5c6423 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr.18c1f413063f3cb5 namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: cdbe39d2-d880-4956-9fd3-28319dfff175 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 5b93aa5d-53e3-4993-ae82-3896cc5c6423 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr.18c1f41306deee5d namespace: kserve-ci-e2e-test resourceVersion: "21284" uid: 8b18aa21-9370-40c9-ae9b-83da488a4302 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T20:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 5b93aa5d-53e3-4993-ae82-3896cc5c6423 kind: Event lastTimestamp: "2026-07-13T20:50:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:46Z" name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr.18c1f41324779925 namespace: kserve-ci-e2e-test resourceVersion: "25871" uid: 525bc7fd-0cda-41ae-9419-fdf034bd84ea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 5b93aa5d-53e3-4993-ae82-3896cc5c6423 kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:44Z" name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr.18c1f4940a979d0f namespace: kserve-ci-e2e-test resourceVersion: "25705" uid: bb204cec-b745-4175-9749-830afabbef32 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 5b93aa5d-53e3-4993-ae82-3896cc5c6423 kind: Event lastTimestamp: "2026-07-13T20:50:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:44Z" name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr.18c1f4940a98419d namespace: kserve-ci-e2e-test resourceVersion: "25707" uid: 9c3289ad-bd96-4d85-a14b-160346696c0f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:50:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 5b93aa5d-53e3-4993-ae82-3896cc5c6423 kind: Event lastTimestamp: "2026-07-13T20:50:46Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:50:46Z" name: success-200-isvc-5859e-predictor-76c758fc77-sq8sr.18c1f49498da88db namespace: kserve-ci-e2e-test resourceVersion: "25868" uid: e047519a-cad7-454a-9016-453d1bdb071a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5859e-predictor-76c758fc77 namespace: kserve-ci-e2e-test resourceVersion: "21177" uid: f046fbda-6bf3-42f3-ba9c-dcf41c950ba6 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: 'Created pod: success-200-isvc-5859e-predictor-76c758fc77-sq8sr' metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-5859e-predictor-76c758fc77.18c1f412e11a0147 namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: 83ec65fd-090e-4064-8d96-0e3ec56e8f43 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5859e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: d4fc3995-7361-4594-90a8-2b40ce88381e kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Scaled up replica set success-200-isvc-5859e-predictor-76c758fc77 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-5859e-predictor.18c1f412df062425 namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: 12bfb84a-2248-4126-bb85-302841b6f27a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:41:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5859e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: ae182c20-b525-4295-ab58-f4feb5e91ce3 kind: Event lastTimestamp: "2026-07-13T20:41:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:59Z" name: success-200-isvc-5859e-predictor.18c1f4165d5b9c84 namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: 0c08032d-b879-42b5-bb21-d9b9c1b03355 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:41:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5859e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: ae182c20-b525-4295-ab58-f4feb5e91ce3 kind: Event lastTimestamp: "2026-07-13T20:41:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:59Z" name: success-200-isvc-5859e-predictor.18c1f4165dc394e9 namespace: kserve-ci-e2e-test resourceVersion: "21551" uid: 0dd9f8e4-ae0b-4f2e-87a8-7c8451364d1f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:42:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5859e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21433" uid: ae182c20-b525-4295-ab58-f4feb5e91ce3 kind: Event lastTimestamp: "2026-07-13T20:42:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:42:44Z" name: success-200-isvc-5859e-predictor.18c1f41d5bf7ae0b namespace: kserve-ci-e2e-test resourceVersion: "21943" uid: 391cff17-4079-4fa0-8f2c-f479c39fa5cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:42:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5859e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21433" uid: ae182c20-b525-4295-ab58-f4feb5e91ce3 kind: Event lastTimestamp: "2026-07-13T20:42:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:42:44Z" name: success-200-isvc-5859e-predictor.18c1f41d5c67fa65 namespace: kserve-ci-e2e-test resourceVersion: "21944" uid: 3db4ac07-344a-44de-bada-365f8d19963d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5859e namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: a04001ca-02be-43d4-9214-075a582b4eec kind: Event lastTimestamp: "2026-07-13T20:42:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:42:16Z" name: success-200-isvc-5859e.18c1f412df0f1dbd namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 917a9df6-ee7f-4d74-9062-be21317f9e65 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5859e namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: a04001ca-02be-43d4-9214-075a582b4eec kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: 'Failed to update status for InferenceService "success-200-isvc-5859e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5859e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-5859e.18c1f412df99754c namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 3b435eca-55bc-4640-9752-3a498af88cef reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5859e namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: a04001ca-02be-43d4-9214-075a582b4eec kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5859e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-5859e.18c1f412df99f1b1 namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: 95395c97-1cbe-43a2-8f60-78f9aa6cfd45 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:42:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5859e namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: a04001ca-02be-43d4-9214-075a582b4eec kind: Event lastTimestamp: "2026-07-13T20:42:16Z" message: InferenceService [success-200-isvc-5859e] is Ready metadata: creationTimestamp: "2026-07-13T20:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:42:16Z" name: success-200-isvc-5859e.18c1f41de4cc616e namespace: kserve-ci-e2e-test resourceVersion: "21704" uid: 1b3f9932-2193-4c71-bbe3-7c915e3567cf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:30:04.289076Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-91263-predictor-867c654bd7-l96fd namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: be140f10-ffd1-4681-9ca0-4f895724bd3f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-91263-predictor-867c654bd7-l96fd to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-91263-predictor-867c654bd7-l96fd.18c1f37360f8018f namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 81e28fd0-ca92-4564-856d-c379e93aa2e8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-91263-predictor-867c654bd7-l96fd namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: be140f10-ffd1-4681-9ca0-4f895724bd3f kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-91263-predictor-867c654bd7-l96fd.18c1f3737b1f9208 namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 30c1be71-9a43-47f5-83a9-85fce4df70a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91263-predictor-867c654bd7-l96fd namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: be140f10-ffd1-4681-9ca0-4f895724bd3f kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-91263-predictor-867c654bd7-l96fd.18c1f3737cdd8f51 namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: 41a6257c-2091-4592-a16c-47b865e695b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91263-predictor-867c654bd7-l96fd namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: be140f10-ffd1-4681-9ca0-4f895724bd3f kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-91263-predictor-867c654bd7-l96fd.18c1f373813df814 namespace: kserve-ci-e2e-test resourceVersion: "15286" uid: e6c9d656-ebaa-4f59-aafb-7531f4395caa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91263-predictor-867c654bd7-l96fd namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: be140f10-ffd1-4681-9ca0-4f895724bd3f kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-91263-predictor-867c654bd7-l96fd.18c1f37381b85890 namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: 57a38273-8851-4be3-a28d-a6dcc2ec09be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-91263-predictor-867c654bd7-l96fd namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: be140f10-ffd1-4681-9ca0-4f895724bd3f kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-91263-predictor-867c654bd7-l96fd.18c1f37381c2317c namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: d3f4ff47-1687-4975-b028-d026aec3b5b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-91263-predictor-867c654bd7-l96fd namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: be140f10-ffd1-4681-9ca0-4f895724bd3f kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-91263-predictor-867c654bd7-l96fd.18c1f373844f8a0c namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: ec57c20c-73ab-4848-b94a-560c1cec7b11 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-91263-predictor-867c654bd7-l96fd namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: be140f10-ffd1-4681-9ca0-4f895724bd3f kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-91263-predictor-867c654bd7-l96fd.18c1f37384d271e2 namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: f4783212-9449-4b17-be60-1730008b6486 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:30:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91263-predictor-867c654bd7-l96fd namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: be140f10-ffd1-4681-9ca0-4f895724bd3f kind: Event lastTimestamp: "2026-07-13T20:30:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:42Z" name: success-200-isvc-91263-predictor-867c654bd7-l96fd.18c1f373ed69b840 namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: eb442bd7-226b-49af-9d5d-a742d93a2560 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91263-predictor-867c654bd7-l96fd namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: be140f10-ffd1-4681-9ca0-4f895724bd3f kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-91263-predictor-867c654bd7-l96fd.18c1f3f49511e4cd namespace: kserve-ci-e2e-test resourceVersion: "19602" uid: 2ad883b8-db9d-40c3-addd-db4167b5d30e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-91263-predictor-867c654bd7-l96fd namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: be140f10-ffd1-4681-9ca0-4f895724bd3f kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:19Z" name: success-200-isvc-91263-predictor-867c654bd7-l96fd.18c1f3f49512dc7c namespace: kserve-ci-e2e-test resourceVersion: "19606" uid: 47c39799-9507-44c7-ad12-41bb65c6cc8f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-91263-predictor-867c654bd7 namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: c27bac66-c852-4ea4-bd59-33d771c3f1a8 kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: 'Created pod: success-200-isvc-91263-predictor-867c654bd7-l96fd' metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-91263-predictor-867c654bd7.18c1f373603e8227 namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: 17ea3455-625f-4ebb-94ec-bbaadb7c1d5d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-91263-predictor namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: 622f96b1-d8d6-4cb2-8db4-db4baacc307f kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: Scaled up replica set success-200-isvc-91263-predictor-867c654bd7 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-91263-predictor.18c1f3735e8242ae namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: e633eed4-bb4a-4c98-841a-12ba337b3e05 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-91263-predictor namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: 89a0d72a-43ad-47e5-b253-bb00e731fb52 kind: Event lastTimestamp: "2026-07-13T20:30:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:19Z" name: success-200-isvc-91263-predictor.18c1f376dd42457b namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: 749409b7-25a0-4a9d-bd1e-2dba7c583010 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-91263-predictor namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: 89a0d72a-43ad-47e5-b253-bb00e731fb52 kind: Event lastTimestamp: "2026-07-13T20:30:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:19Z" name: success-200-isvc-91263-predictor.18c1f376ddbe1053 namespace: kserve-ci-e2e-test resourceVersion: "15439" uid: 49da77b5-b61e-4ce6-9c93-619c1e0fde85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:30:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-91263-predictor namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: 89a0d72a-43ad-47e5-b253-bb00e731fb52 kind: Event lastTimestamp: "2026-07-13T20:31:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:19Z" name: success-200-isvc-91263-predictor.18c1f37a5c8b9661 namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: 7d037200-12bd-4c3e-abda-537759330e23 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T20:30:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-91263-predictor namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: 89a0d72a-43ad-47e5-b253-bb00e731fb52 kind: Event lastTimestamp: "2026-07-13T20:31:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:19Z" name: success-200-isvc-91263-predictor.18c1f37a5d070e25 namespace: kserve-ci-e2e-test resourceVersion: "16205" uid: 154882cc-737b-43dd-aae2-fdf6aaf5ce68 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-91263 namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: 66f3e08f-0305-4db6-8f32-6cc495d36f9c kind: Event lastTimestamp: "2026-07-13T20:30:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:52Z" name: success-200-isvc-91263.18c1f3735e83cd7d namespace: kserve-ci-e2e-test resourceVersion: "15750" uid: e3cdac68-6a55-4daa-bec8-c9121c161ef8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-91263 namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: 66f3e08f-0305-4db6-8f32-6cc495d36f9c kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-91263": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-91263": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-91263.18c1f3735f1fd462 namespace: kserve-ci-e2e-test resourceVersion: "15225" uid: a5757b9b-dc70-475a-9331-765352304415 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-91263 namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: 66f3e08f-0305-4db6-8f32-6cc495d36f9c kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-91263": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-91263.18c1f3735f2024df namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: 1812a13f-d747-40c9-ab89-4d4882b25f1c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-91263 namespace: kserve-ci-e2e-test resourceVersion: "15747" uid: 66f3e08f-0305-4db6-8f32-6cc495d36f9c kind: Event lastTimestamp: "2026-07-13T20:30:52Z" message: InferenceService [success-200-isvc-91263] is Ready metadata: creationTimestamp: "2026-07-13T20:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:30:52Z" name: success-200-isvc-91263.18c1f37ea9b97a01 namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: 77c76d60-2eef-4be4-a30c-3f443ee14f2e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:28:34.033006Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14170" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-cbd18-predictor-749f5f5477-xmgml to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:28:34Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f35e5d49cad7 namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: 8096fc6f-3bf7-4e9b-ac6c-3bc5058f9469 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14179" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:28:34Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f35e78bf3e1d namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: ee19bd50-8c31-4b29-a73b-e4e747bf4679 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1733" metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:34Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f35e7a761e2e namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: 08d0fd79-562e-4566-9b6b-c95c27a7cbed reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: "2026-07-13T20:28:50Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1733" in 15.677s (15.677s including waiting). Image size: 1335613554 bytes.' metadata: creationTimestamp: "2026-07-13T20:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:50Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f36220ee64c4 namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 18f9495f-bb2e-4c08-b659-8776fe76fb93 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: "2026-07-13T20:28:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:50Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f36224e5eb45 namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: d427ba47-d3ce-42eb-97e0-10cc7ebd41f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: "2026-07-13T20:28:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:50Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f362255ed2e8 namespace: kserve-ci-e2e-test resourceVersion: "14464" uid: f5d0fbd8-316a-4eb3-af67-0e61c1a33118 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: "2026-07-13T20:28:50Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-13T20:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:50Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f362256d2579 namespace: kserve-ci-e2e-test resourceVersion: "14465" uid: d3dbd5c7-d58e-48e2-9af1-e5780cf8ecc5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: "2026-07-13T20:28:52Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.679s (2.679s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-13T20:28:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:52Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f362c51f8eb9 namespace: kserve-ci-e2e-test resourceVersion: "14484" uid: 01e34034-7004-4f8d-9794-3029262dcfb0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: "2026-07-13T20:28:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:53Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f362c8424a89 namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: 9beab3ce-0410-4662-a980-1353f5e8b643 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: "2026-07-13T20:28:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:28:53Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f362c8bb6a96 namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: a69ab057-3961-4ddf-914c-7c9e7802fe6f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: "2026-07-13T20:29:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:29Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f362dd95c8d0 namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: d8640cf4-5d8a-4450-b1dd-638b23381f10 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f3735c6a36ae namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 1e846de6-6450-4691-97d5-39335f266a9a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f3735c6acb04 namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 16ae762a-d2a3-4ceb-b589-12fb89f3aa44 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b7d4fc3c-cd56-4763-a3bb-aab8d81f2292 kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-cbd18-predictor-749f5f5477-xmgml.18c1f37365d75d74 namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: b5bd8f45-2e69-426c-81fa-f3c751dcae35 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-cbd18-predictor-749f5f5477 namespace: kserve-ci-e2e-test resourceVersion: "14162" uid: 3e2280f7-faf7-4d4f-9b6f-86ef43e489aa kind: Event lastTimestamp: "2026-07-13T20:28:34Z" message: 'Created pod: success-200-isvc-cbd18-predictor-749f5f5477-xmgml' metadata: creationTimestamp: "2026-07-13T20:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:28:34Z" name: success-200-isvc-cbd18-predictor-749f5f5477.18c1f35e5ccca09b namespace: kserve-ci-e2e-test resourceVersion: "14172" uid: 132f41fb-078f-43b5-999c-8b1154ddd0d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:28:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-cbd18-predictor namespace: kserve-ci-e2e-test resourceVersion: "14160" uid: 093a38ef-38fa-4bf3-bb00-7bd439f9b5f6 kind: Event lastTimestamp: "2026-07-13T20:28:33Z" message: Scaled up replica set success-200-isvc-cbd18-predictor-749f5f5477 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:28:33Z" name: success-200-isvc-cbd18-predictor.18c1f35e5a6bbff5 namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: a70861c3-45d0-4f1d-aebd-57461a7e87e3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:28:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cbd18-predictor namespace: kserve-ci-e2e-test resourceVersion: "14161" uid: d863ff25-fb22-4fa6-b974-8bba6d8aeeeb kind: Event lastTimestamp: "2026-07-13T20:29:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:34Z" name: success-200-isvc-cbd18-predictor.18c1f361d8d3d619 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: 2c280180-8df2-4870-8bd9-3080cfb5b342 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:28:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cbd18-predictor namespace: kserve-ci-e2e-test resourceVersion: "14161" uid: d863ff25-fb22-4fa6-b974-8bba6d8aeeeb kind: Event lastTimestamp: "2026-07-13T20:29:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:34Z" name: success-200-isvc-cbd18-predictor.18c1f361d9402113 namespace: kserve-ci-e2e-test resourceVersion: "14854" uid: 33068647-0df2-4004-b3e7-20840815a93f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:29:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cbd18-predictor namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: d863ff25-fb22-4fa6-b974-8bba6d8aeeeb kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-cbd18-predictor.18c1f368d6d83ea9 namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: dd9bd69f-6cf5-4c05-89ef-db122782a170 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:29:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cbd18-predictor namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: d863ff25-fb22-4fa6-b974-8bba6d8aeeeb kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:30:04Z" name: success-200-isvc-cbd18-predictor.18c1f368d74bf777 namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: 400da891-9fdf-415a-ab8a-3a1581791055 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:28:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cbd18 namespace: kserve-ci-e2e-test resourceVersion: "14144" uid: 4dcbfa6d-e3e0-4171-9843-0ae272b0c112 kind: Event lastTimestamp: "2026-07-13T20:29:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:29:39Z" name: success-200-isvc-cbd18.18c1f35e5aa5bbf8 namespace: kserve-ci-e2e-test resourceVersion: "14914" uid: 3e9511f5-7411-40ae-aa9e-4015c75aa146 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cbd18 namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: 4dcbfa6d-e3e0-4171-9843-0ae272b0c112 kind: Event lastTimestamp: "2026-07-13T20:29:39Z" message: InferenceService [success-200-isvc-cbd18] is Ready metadata: creationTimestamp: "2026-07-13T20:29:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:29:39Z" name: success-200-isvc-cbd18.18c1f36d99f45b65 namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: dfab9205-78c2-465f-9e5b-a4638a04aeb1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:40:29.623226Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: a29df3fa-bea3-4590-8ecf-151bc07970c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:40:29Z" name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2.18c1f404f9c9e00f namespace: kserve-ci-e2e-test resourceVersion: "20486" uid: 88171f25-84d3-4e31-b4b6-56f76028f51e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: a29df3fa-bea3-4590-8ecf-151bc07970c9 kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-db27e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:29Z" name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2.18c1f405038fd76f namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: abb8cda7-20fd-4346-a425-6e2b9ebad0b5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: a29df3fa-bea3-4590-8ecf-151bc07970c9 kind: Event lastTimestamp: "2026-07-13T20:40:30Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:40:30Z" name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2.18c1f40538b1e24d namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: ac2c4400-9e0b-4018-8c6b-43fd472f96db reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: a29df3fa-bea3-4590-8ecf-151bc07970c9 kind: Event lastTimestamp: "2026-07-13T20:40:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:30Z" name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2.18c1f4053ab4d674 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: e9aa3b65-d336-4a6b-a2f3-a34809112c49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: a29df3fa-bea3-4590-8ecf-151bc07970c9 kind: Event lastTimestamp: "2026-07-13T20:40:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:30Z" name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2.18c1f4053edb9dbc namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: ab310c16-8e28-44c1-8ea4-49516937dc0d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: a29df3fa-bea3-4590-8ecf-151bc07970c9 kind: Event lastTimestamp: "2026-07-13T20:40:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:30Z" name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2.18c1f4053f846b9f namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: 3d379327-b793-46e1-82dd-32b8d0ec8e21 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: a29df3fa-bea3-4590-8ecf-151bc07970c9 kind: Event lastTimestamp: "2026-07-13T20:40:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:30Z" name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2.18c1f4053f93beb6 namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: adeb5884-469b-48ab-93af-1a794bc5ebbd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: a29df3fa-bea3-4590-8ecf-151bc07970c9 kind: Event lastTimestamp: "2026-07-13T20:40:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:30Z" name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2.18c1f4054388889f namespace: kserve-ci-e2e-test resourceVersion: "20566" uid: b429dca6-67b4-490f-a9b0-ac0c0aa946a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: a29df3fa-bea3-4590-8ecf-151bc07970c9 kind: Event lastTimestamp: "2026-07-13T20:40:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:30Z" name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2.18c1f405440cb4a8 namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: a9c505b9-f45a-4e47-8912-879944906f84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T20:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: a29df3fa-bea3-4590-8ecf-151bc07970c9 kind: Event lastTimestamp: "2026-07-13T20:49:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:47Z" name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2.18c1f40557e6b8f0 namespace: kserve-ci-e2e-test resourceVersion: "24939" uid: acf860a6-dd74-476a-ab83-feca85b6128f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: a29df3fa-bea3-4590-8ecf-151bc07970c9 kind: Event lastTimestamp: "2026-07-13T20:49:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:44Z" name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2.18c1f4862a7fcdea namespace: kserve-ci-e2e-test resourceVersion: "24823" uid: ef8139b2-e18a-4f73-bdc3-a6d4c61eb31f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: a29df3fa-bea3-4590-8ecf-151bc07970c9 kind: Event lastTimestamp: "2026-07-13T20:49:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:44Z" name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2.18c1f4862a80817a namespace: kserve-ci-e2e-test resourceVersion: "24825" uid: d82552f7-bbb1-4156-b1de-e9b2c5a422e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:49:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: a29df3fa-bea3-4590-8ecf-151bc07970c9 kind: Event lastTimestamp: "2026-07-13T20:49:47Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:49:47Z" name: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2.18c1f486cc4feb14 namespace: kserve-ci-e2e-test resourceVersion: "24937" uid: 8f07c43a-d7d8-4f40-9581-9be5b56e829c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-db27e-predictor-6c64cf5797 namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: b309e9b1-961d-4352-b878-993881d22633 kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: 'Created pod: success-200-isvc-db27e-predictor-6c64cf5797-l4fz2' metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:29Z" name: success-200-isvc-db27e-predictor-6c64cf5797.18c1f404f918023c namespace: kserve-ci-e2e-test resourceVersion: "20482" uid: 5d80ed9d-407a-460d-aed1-9dfc9be68528 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-db27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: 51727a96-30bf-4e2d-aef4-ca7625aeb2ba kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: Scaled up replica set success-200-isvc-db27e-predictor-6c64cf5797 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:29Z" name: success-200-isvc-db27e-predictor.18c1f404f67110ba namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 2d42f5ac-419c-4ab6-b906-87987aef9a3f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:40:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: c5396a98-7e51-4554-a66e-5c363bbc8905 kind: Event lastTimestamp: "2026-07-13T20:40:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:59Z" name: success-200-isvc-db27e-predictor.18c1f40874a3d3bc namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: cbaef3b0-5d28-42cc-a731-352ee3d185d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:40:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: c5396a98-7e51-4554-a66e-5c363bbc8905 kind: Event lastTimestamp: "2026-07-13T20:40:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:59Z" name: success-200-isvc-db27e-predictor.18c1f40875737f54 namespace: kserve-ci-e2e-test resourceVersion: "20831" uid: fe9460e9-6829-4c48-8f53-2c168eaa95c2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:41:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: c5396a98-7e51-4554-a66e-5c363bbc8905 kind: Event lastTimestamp: "2026-07-13T20:41:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:41:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:44Z" name: success-200-isvc-db27e-predictor.18c1f40f7368b7dd namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: 380d97d0-4b2a-445a-b203-8436108ae0d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:41:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: c5396a98-7e51-4554-a66e-5c363bbc8905 kind: Event lastTimestamp: "2026-07-13T20:41:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:41:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:44Z" name: success-200-isvc-db27e-predictor.18c1f40f73ebe5c9 namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: 2d0ca541-156c-4cac-ab1b-706e741959b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db27e namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 4d44486e-c455-43ff-b519-ec3c3625a7b7 kind: Event lastTimestamp: "2026-07-13T20:41:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:17Z" name: success-200-isvc-db27e.18c1f404f63e5bca namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: c4f07b51-4668-41cc-af5d-042b6a4e79b3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db27e namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 4d44486e-c455-43ff-b519-ec3c3625a7b7 kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: 'Failed to update status for InferenceService "success-200-isvc-db27e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-db27e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:40:29Z" name: success-200-isvc-db27e.18c1f404f6b7451b namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: 3ce0668b-7e56-405f-a435-cbc3c4b36164 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db27e namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 4d44486e-c455-43ff-b519-ec3c3625a7b7 kind: Event lastTimestamp: "2026-07-13T20:40:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-db27e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:40:29Z" name: success-200-isvc-db27e.18c1f404f6b79c1f namespace: kserve-ci-e2e-test resourceVersion: "20477" uid: 0befb22f-8da3-48d8-9bd2-0e7096460d77 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db27e namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: 4d44486e-c455-43ff-b519-ec3c3625a7b7 kind: Event lastTimestamp: "2026-07-13T20:41:17Z" message: InferenceService [success-200-isvc-db27e] is Ready metadata: creationTimestamp: "2026-07-13T20:41:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:17Z" name: success-200-isvc-db27e.18c1f4101c079705 namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: 71ffe472-59b3-46fd-9da6-5d0df657cf49 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:40:19.008325Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e8943-predictor-59544c7895-2k9ld namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: e577908b-ec46-4774-83da-4511471d7557 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e8943-predictor-59544c7895-2k9ld to ip-10-0-128-233.ec2.internal metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:40:19Z" name: success-200-isvc-e8943-predictor-59544c7895-2k9ld.18c1f40281176b08 namespace: kserve-ci-e2e-test resourceVersion: "20215" uid: 3820fdd3-1f18-4167-891b-e4e4a97d8f16 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e8943-predictor-59544c7895-2k9ld namespace: kserve-ci-e2e-test resourceVersion: "20220" uid: e577908b-ec46-4774-83da-4511471d7557 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:40:19Z" name: success-200-isvc-e8943-predictor-59544c7895-2k9ld.18c1f4029b28e5d2 namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: e3383af2-9090-4114-a494-5e89b4218504 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8943-predictor-59544c7895-2k9ld namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: e577908b-ec46-4774-83da-4511471d7557 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1733" already present on machine metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: success-200-isvc-e8943-predictor-59544c7895-2k9ld.18c1f4029d1f45ca namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 341a4af9-e3e1-4271-a735-e5d0c43e762c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8943-predictor-59544c7895-2k9ld namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: e577908b-ec46-4774-83da-4511471d7557 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: success-200-isvc-e8943-predictor-59544c7895-2k9ld.18c1f402a1058d8b namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: 10f5b227-355e-4e8f-b579-63b866b3084a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8943-predictor-59544c7895-2k9ld namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: e577908b-ec46-4774-83da-4511471d7557 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: success-200-isvc-e8943-predictor-59544c7895-2k9ld.18c1f402a1973e93 namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 31d6a3cf-2609-4515-bfff-5863961ccf66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e8943-predictor-59544c7895-2k9ld namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: e577908b-ec46-4774-83da-4511471d7557 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: success-200-isvc-e8943-predictor-59544c7895-2k9ld.18c1f402a1a38e2a namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: b1a4d9b8-435d-4fad-bbb1-5f50a5993891 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e8943-predictor-59544c7895-2k9ld namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: e577908b-ec46-4774-83da-4511471d7557 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: success-200-isvc-e8943-predictor-59544c7895-2k9ld.18c1f402a604a425 namespace: kserve-ci-e2e-test resourceVersion: "20315" uid: 730ddd22-e7bd-40dc-be9b-13f6aac2c060 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e8943-predictor-59544c7895-2k9ld namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: e577908b-ec46-4774-83da-4511471d7557 kind: Event lastTimestamp: "2026-07-13T20:40:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:19Z" name: success-200-isvc-e8943-predictor-59544c7895-2k9ld.18c1f402a67ff658 namespace: kserve-ci-e2e-test resourceVersion: "20318" uid: 62e47555-f113-46f0-b604-12c3e7fa8ce4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8943-predictor-59544c7895-2k9ld namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: e577908b-ec46-4774-83da-4511471d7557 kind: Event lastTimestamp: "2026-07-13T20:40:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:40:56Z" name: success-200-isvc-e8943-predictor-59544c7895-2k9ld.18c1f402c4938ea8 namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: bf8dec70-4c29-46c9-b57e-caa745a45d5c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8943-predictor-59544c7895-2k9ld namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: e577908b-ec46-4774-83da-4511471d7557 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-e8943-predictor-59544c7895-2k9ld.18c1f412dbf50bcb namespace: kserve-ci-e2e-test resourceVersion: "21161" uid: c9ce361f-ec21-4e69-99b2-3995fe2f3dd5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e8943-predictor-59544c7895-2k9ld namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: e577908b-ec46-4774-83da-4511471d7557 kind: Event lastTimestamp: "2026-07-13T20:41:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T20:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:29Z" name: success-200-isvc-e8943-predictor-59544c7895-2k9ld.18c1f412dbf5afad namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 62e3cd8b-ecb1-49f3-87fc-322284fa36f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e8943-predictor-59544c7895-2k9ld namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: e577908b-ec46-4774-83da-4511471d7557 kind: Event lastTimestamp: "2026-07-13T20:41:31Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T20:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:41:31Z" name: success-200-isvc-e8943-predictor-59544c7895-2k9ld.18c1f4134ce1e461 namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: b66b6ba4-4ac9-44ac-be59-eeb0422e2ad9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-233.ec2.internal source: component: kubelet host: ip-10-0-128-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e8943-predictor-59544c7895 namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 4917d6b3-c347-437d-aa25-01791f51c47a kind: Event lastTimestamp: "2026-07-13T20:40:18Z" message: 'Created pod: success-200-isvc-e8943-predictor-59544c7895-2k9ld' metadata: creationTimestamp: "2026-07-13T20:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:18Z" name: success-200-isvc-e8943-predictor-59544c7895.18c1f40280829b8f namespace: kserve-ci-e2e-test resourceVersion: "20212" uid: 91a47e14-7fbe-4198-bc5c-161f2b725134 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e8943-predictor namespace: kserve-ci-e2e-test resourceVersion: "20189" uid: e7ec7809-ebf7-44c0-b91a-bbb5e93188af kind: Event lastTimestamp: "2026-07-13T20:40:18Z" message: Scaled up replica set success-200-isvc-e8943-predictor-59544c7895 from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:40:18Z" name: success-200-isvc-e8943-predictor.18c1f4027daf3f81 namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: a13fe951-a763-4e86-a596-ecf6e8e1c416 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:40:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e8943-predictor namespace: kserve-ci-e2e-test resourceVersion: "20191" uid: 7478cb11-e6bc-4395-880e-a194f584c5d4 kind: Event lastTimestamp: "2026-07-13T20:41:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:04Z" name: success-200-isvc-e8943-predictor.18c1f405fcab313e namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 3977a6ec-a075-4ad5-8ca6-15b345fe6553 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T20:40:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e8943-predictor namespace: kserve-ci-e2e-test resourceVersion: "20191" uid: 7478cb11-e6bc-4395-880e-a194f584c5d4 kind: Event lastTimestamp: "2026-07-13T20:41:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:04Z" name: success-200-isvc-e8943-predictor.18c1f405fd2a8e71 namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: 7e4d6d80-f5bf-4580-88ab-befbe1bb5e61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e8943-predictor namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 7478cb11-e6bc-4395-880e-a194f584c5d4 kind: Event lastTimestamp: "2026-07-13T20:41:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:19Z" name: success-200-isvc-e8943-predictor.18c1f4107ab3bf53 namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 0cb30235-1686-405d-92ac-4c764a005cb1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e8943-predictor namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 7478cb11-e6bc-4395-880e-a194f584c5d4 kind: Event lastTimestamp: "2026-07-13T20:41:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:41:19Z" name: success-200-isvc-e8943-predictor.18c1f4107b6922f9 namespace: kserve-ci-e2e-test resourceVersion: "21027" uid: 17487fd6-dadd-480f-8a1c-87618c6d3a63 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T20:40:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e8943 namespace: kserve-ci-e2e-test resourceVersion: "20176" uid: 525a0ada-b34e-4322-aa66-8eb73f9d1b4e kind: Event lastTimestamp: "2026-07-13T20:41:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T20:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:06Z" name: success-200-isvc-e8943.18c1f4027de81e89 namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: 86585e62-ec48-46b1-a17d-19f649488892 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e8943 namespace: kserve-ci-e2e-test resourceVersion: "20176" uid: 525a0ada-b34e-4322-aa66-8eb73f9d1b4e kind: Event lastTimestamp: "2026-07-13T20:40:18Z" message: 'Failed to update status for InferenceService "success-200-isvc-e8943": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e8943": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:40:18Z" name: success-200-isvc-e8943.18c1f4027e8c3f51 namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: 64e24683-4014-4a8b-9834-20c5affd278a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:40:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e8943 namespace: kserve-ci-e2e-test resourceVersion: "20176" uid: 525a0ada-b34e-4322-aa66-8eb73f9d1b4e kind: Event lastTimestamp: "2026-07-13T20:40:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e8943": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:40:18Z" name: success-200-isvc-e8943.18c1f4027e8c8b2c namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 15c1d68b-3e8e-48a2-b7bd-87d75ed33b6b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:41:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e8943 namespace: kserve-ci-e2e-test resourceVersion: "20898" uid: 525a0ada-b34e-4322-aa66-8eb73f9d1b4e kind: Event lastTimestamp: "2026-07-13T20:41:06Z" message: InferenceService [success-200-isvc-e8943] is Ready metadata: creationTimestamp: "2026-07-13T20:41:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:41:06Z" name: success-200-isvc-e8943.18c1f40d84d113fc namespace: kserve-ci-e2e-test resourceVersion: "20899" uid: 0f827e64-63ab-49c5-8f39-da8379df8cbc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:51:54.237078Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-575ad-64d456d77c-rsdgn namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: e36ee415-fdf7-4b2c-931a-7f3da8cc3e06 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-575ad-64d456d77c-rsdgn to ip-10-0-134-97.ec2.internal metadata: creationTimestamp: "2026-07-13T20:51:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:51:54Z" name: switch-graph-575ad-64d456d77c-rsdgn.18c1f4a45ff449ec namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: d71655f0-1b71-4b41-bb6b-c71d92f44d32 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:54Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-575ad-64d456d77c-rsdgn namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: e36ee415-fdf7-4b2c-931a-7f3da8cc3e06 kind: Event lastTimestamp: "2026-07-13T20:51:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-575ad-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:51:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:51:54Z" name: switch-graph-575ad-64d456d77c-rsdgn.18c1f4a46f0b8686 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: ae6a9a30-4472-4008-805e-a7b3bb04e859 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:55Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-575ad-64d456d77c-rsdgn namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: e36ee415-fdf7-4b2c-931a-7f3da8cc3e06 kind: Event lastTimestamp: "2026-07-13T20:51:55Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:51:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:51:55Z" name: switch-graph-575ad-64d456d77c-rsdgn.18c1f4a49dcbc0a3 namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 1c2de422-8cd8-4d14-95bc-da3dbd10122f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-575ad} kind: Pod name: switch-graph-575ad-64d456d77c-rsdgn namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: e36ee415-fdf7-4b2c-931a-7f3da8cc3e06 kind: Event lastTimestamp: "2026-07-13T20:51:55Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:8345cd1e0b8ad942d68e2fe65c99d692fac6452f8ea219efb6700aaf29b3e2fa" already present on machine metadata: creationTimestamp: "2026-07-13T20:51:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:51:55Z" name: switch-graph-575ad-64d456d77c-rsdgn.18c1f4a49f0fe011 namespace: kserve-ci-e2e-test resourceVersion: "26693" uid: 319c892b-6b4f-41fb-966b-aec12c832c8c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-575ad} kind: Pod name: switch-graph-575ad-64d456d77c-rsdgn namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: e36ee415-fdf7-4b2c-931a-7f3da8cc3e06 kind: Event lastTimestamp: "2026-07-13T20:51:55Z" message: 'Created container: switch-graph-575ad' metadata: creationTimestamp: "2026-07-13T20:51:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:51:55Z" name: switch-graph-575ad-64d456d77c-rsdgn.18c1f4a4a1c3d945 namespace: kserve-ci-e2e-test resourceVersion: "26694" uid: 479be9bd-423f-4be9-9f01-f435c320b548 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-575ad} kind: Pod name: switch-graph-575ad-64d456d77c-rsdgn namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: e36ee415-fdf7-4b2c-931a-7f3da8cc3e06 kind: Event lastTimestamp: "2026-07-13T20:51:55Z" message: Started container switch-graph-575ad metadata: creationTimestamp: "2026-07-13T20:51:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:51:55Z" name: switch-graph-575ad-64d456d77c-rsdgn.18c1f4a4a23a1a6b namespace: kserve-ci-e2e-test resourceVersion: "26696" uid: 5ba6bc82-a89a-4a2d-90f3-5d6adbf5c5d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T21:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-575ad} kind: Pod name: switch-graph-575ad-64d456d77c-rsdgn namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: e36ee415-fdf7-4b2c-931a-7f3da8cc3e06 kind: Event lastTimestamp: "2026-07-13T21:08:13Z" message: Stopping container switch-graph-575ad metadata: creationTimestamp: "2026-07-13T21:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:08:13Z" name: switch-graph-575ad-64d456d77c-rsdgn.18c1f588682066b3 namespace: kserve-ci-e2e-test resourceVersion: "33502" uid: af1a03cb-3660-4238-bf4e-16ff196d577a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T21:08:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-575ad} kind: Pod name: switch-graph-575ad-64d456d77c-rsdgn namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: e36ee415-fdf7-4b2c-931a-7f3da8cc3e06 kind: Event lastTimestamp: "2026-07-13T21:08:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T21:08:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T21:08:36Z" name: switch-graph-575ad-64d456d77c-rsdgn.18c1f589264aa634 namespace: kserve-ci-e2e-test resourceVersion: "33742" uid: 750fb888-d70a-4f30-b581-e11c003026cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-575ad-64d456d77c namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: f8cb01c0-ce7c-49b1-9f73-d02ab74f01ef kind: Event lastTimestamp: "2026-07-13T20:51:54Z" message: 'Created pod: switch-graph-575ad-64d456d77c-rsdgn' metadata: creationTimestamp: "2026-07-13T20:51:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:54Z" name: switch-graph-575ad-64d456d77c.18c1f4a45f89c3e1 namespace: kserve-ci-e2e-test resourceVersion: "26633" uid: ec4873ad-945e-4c65-8678-6194195d62be reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-575ad namespace: kserve-ci-e2e-test resourceVersion: "26625" uid: f5238e84-cd79-43c8-9821-bdba872a784c kind: Event lastTimestamp: "2026-07-13T20:51:54Z" message: Scaled up replica set switch-graph-575ad-64d456d77c from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:51:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:51:54Z" name: switch-graph-575ad.18c1f4a45e9c0102 namespace: kserve-ci-e2e-test resourceVersion: "26630" uid: 1d3063ec-aa24-4680-a495-223c14e8d6ed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:51:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-575ad namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 92a0fbb4-4487-4312-bcac-d78b01b5f6da kind: Event lastTimestamp: "2026-07-13T20:51:54Z" message: InferenceGraph [switch-graph-575ad] is Ready metadata: creationTimestamp: "2026-07-13T20:51:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:51:54Z" name: switch-graph-575ad.18c1f4a4634f72a0 namespace: kserve-ci-e2e-test resourceVersion: "26645" uid: 325339c3-2d1c-4752-b3d6-596e57dc6cac reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:52:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-575ad namespace: kserve-ci-e2e-test resourceVersion: "26626" uid: 2daedaa7-7a1f-4f5e-a07a-c0d2cf3a5740 kind: Event lastTimestamp: "2026-07-13T20:52:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:52:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:52:09Z" name: switch-graph-575ad.18c1f4a7dd160b53 namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: c24b1f91-33b9-48e4-aefa-cd98b63dbc64 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:52:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-575ad namespace: kserve-ci-e2e-test resourceVersion: "26626" uid: 2daedaa7-7a1f-4f5e-a07a-c0d2cf3a5740 kind: Event lastTimestamp: "2026-07-13T20:52:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:52:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:52:09Z" name: switch-graph-575ad.18c1f4a7dd6c6124 namespace: kserve-ci-e2e-test resourceVersion: "26814" uid: f4dc5e07-c213-4675-8260-45135f0eb282 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:52:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-575ad namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: 2daedaa7-7a1f-4f5e-a07a-c0d2cf3a5740 kind: Event lastTimestamp: "2026-07-13T20:52:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:52:24Z" name: switch-graph-575ad.18c1f4ab5c8ad4a4 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 8a0693fd-c600-496e-b9d0-fafb006a98b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:52:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-575ad namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: 2daedaa7-7a1f-4f5e-a07a-c0d2cf3a5740 kind: Event lastTimestamp: "2026-07-13T20:52:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T20:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:52:24Z" name: switch-graph-575ad.18c1f4ab5cf9a105 namespace: kserve-ci-e2e-test resourceVersion: "26927" uid: 94a9b67f-59f3-4bd2-9fe0-63621ff58a84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:31:04.344817Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-91263-5f5cb88ccb-cqznj namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: cbfa2e52-3b47-4cb0-9471-3c6170494b8d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-91263-5f5cb88ccb-cqznj to ip-10-0-134-97.ec2.internal metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:31:04Z" name: switch-graph-91263-5f5cb88ccb-cqznj.18c1f3815c91fc27 namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: caa661cc-bd21-4bb2-8ba9-82d01bca5105 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-91263-5f5cb88ccb-cqznj namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: cbfa2e52-3b47-4cb0-9471-3c6170494b8d kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:31:04Z" name: switch-graph-91263-5f5cb88ccb-cqznj.18c1f3817614bb91 namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: 0c09120f-0f35-4793-af30-ea3619072d1d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-91263} kind: Pod name: switch-graph-91263-5f5cb88ccb-cqznj namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: cbfa2e52-3b47-4cb0-9471-3c6170494b8d kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:8345cd1e0b8ad942d68e2fe65c99d692fac6452f8ea219efb6700aaf29b3e2fa" already present on machine metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:04Z" name: switch-graph-91263-5f5cb88ccb-cqznj.18c1f38177b0423d namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: c2e444e0-45cc-417b-86e2-c6640af9fe1b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-91263} kind: Pod name: switch-graph-91263-5f5cb88ccb-cqznj namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: cbfa2e52-3b47-4cb0-9471-3c6170494b8d kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: 'Created container: switch-graph-91263' metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:04Z" name: switch-graph-91263-5f5cb88ccb-cqznj.18c1f3817a458a67 namespace: kserve-ci-e2e-test resourceVersion: "16041" uid: 174d6160-9612-4577-b155-7520a5b1d023 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-91263} kind: Pod name: switch-graph-91263-5f5cb88ccb-cqznj namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: cbfa2e52-3b47-4cb0-9471-3c6170494b8d kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: Started container switch-graph-91263 metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:31:04Z" name: switch-graph-91263-5f5cb88ccb-cqznj.18c1f3817ab68970 namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: d54dec28-4948-4b59-a1f7-d81bd69a8ad6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-91263} kind: Pod name: switch-graph-91263-5f5cb88ccb-cqznj namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: cbfa2e52-3b47-4cb0-9471-3c6170494b8d kind: Event lastTimestamp: "2026-07-13T20:39:19Z" message: Stopping container switch-graph-91263 metadata: creationTimestamp: "2026-07-13T20:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:19Z" name: switch-graph-91263-5f5cb88ccb-cqznj.18c1f3f48a0d4ac9 namespace: kserve-ci-e2e-test resourceVersion: "19571" uid: 6bbd29d6-a0c8-49d8-a459-44113dcbb35d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:39:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-91263} kind: Pod name: switch-graph-91263-5f5cb88ccb-cqznj namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: cbfa2e52-3b47-4cb0-9471-3c6170494b8d kind: Event lastTimestamp: "2026-07-13T20:39:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T20:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:39:45Z" name: switch-graph-91263-5f5cb88ccb-cqznj.18c1f3f4fabcc800 namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: 77b9ead2-5e80-481f-98dc-628cd3818847 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-91263-5f5cb88ccb namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 40905d53-464f-4d2e-a6bd-2fe390b28f09 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: 'Created pod: switch-graph-91263-5f5cb88ccb-cqznj' metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:04Z" name: switch-graph-91263-5f5cb88ccb.18c1f3815c1a17a7 namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: eb76c4ce-43eb-4cbe-bd04-9045949b1fef reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-91263 namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: a9bdbac8-0d9e-4b6b-9454-8bde6001e6e7 kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: Scaled up replica set switch-graph-91263-5f5cb88ccb from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:04Z" name: switch-graph-91263.18c1f3815b431df1 namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: 721e11f3-0326-408c-aa91-1c6d6b0d5c86 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:31:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-91263 namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: cfe16b5f-e2be-49e2-bacd-310e14096fdb kind: Event lastTimestamp: "2026-07-13T20:31:04Z" message: InferenceGraph [switch-graph-91263] is Ready metadata: creationTimestamp: "2026-07-13T20:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:31:04Z" name: switch-graph-91263.18c1f381614bb732 namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: e11a6338-603e-412e-9fc5-9636cab72aef reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:31:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-91263 namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: e52e555e-23b0-46cf-9a07-790662a325db kind: Event lastTimestamp: "2026-07-13T20:31:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:31:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:34Z" name: switch-graph-91263.18c1f384d9b56f1a namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: d2d34636-9412-4d58-9187-9fa6f030d552 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T20:31:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-91263 namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: e52e555e-23b0-46cf-9a07-790662a325db kind: Event lastTimestamp: "2026-07-13T20:31:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T20:31:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:31:34Z" name: switch-graph-91263.18c1f384da2c93fc namespace: kserve-ci-e2e-test resourceVersion: "16323" uid: c7c318c8-21e3-4148-a98e-0997a42418a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-13T20:29:53.902725Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-cbd18-7b6659b66b-s74jh namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: 65a76ec0-4a2a-455f-9c09-638cb6e3986f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-cbd18-7b6659b66b-s74jh to ip-10-0-134-97.ec2.internal metadata: creationTimestamp: "2026-07-13T20:29:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T20:29:53Z" name: switch-graph-cbd18-7b6659b66b-s74jh.18c1f370f5e4f518 namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: 93769c20-5649-44aa-a8e5-17342480cbff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674c9f7b64-fwgd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:54Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-cbd18-7b6659b66b-s74jh namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 65a76ec0-4a2a-455f-9c09-638cb6e3986f kind: Event lastTimestamp: "2026-07-13T20:29:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-cbd18-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T20:29:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:54Z" name: switch-graph-cbd18-7b6659b66b-s74jh.18c1f3710273af6c namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: a2e201c3-8984-4672-8afc-e70f4e90a9c2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:54Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-cbd18-7b6659b66b-s74jh namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: 65a76ec0-4a2a-455f-9c09-638cb6e3986f kind: Event lastTimestamp: "2026-07-13T20:29:54Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T20:29:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T20:29:54Z" name: switch-graph-cbd18-7b6659b66b-s74jh.18c1f37132eef008 namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: f7619f94-7050-4c55-9d57-2181e87d71d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cbd18} kind: Pod name: switch-graph-cbd18-7b6659b66b-s74jh namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 65a76ec0-4a2a-455f-9c09-638cb6e3986f kind: Event lastTimestamp: "2026-07-13T20:29:54Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:8345cd1e0b8ad942d68e2fe65c99d692fac6452f8ea219efb6700aaf29b3e2fa" metadata: creationTimestamp: "2026-07-13T20:29:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:54Z" name: switch-graph-cbd18-7b6659b66b-s74jh.18c1f371344b910f namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 49cd57c4-a73f-46f9-9662-c394aeb95ec6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cbd18} kind: Pod name: switch-graph-cbd18-7b6659b66b-s74jh namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 65a76ec0-4a2a-455f-9c09-638cb6e3986f kind: Event lastTimestamp: "2026-07-13T20:29:57Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:8345cd1e0b8ad942d68e2fe65c99d692fac6452f8ea219efb6700aaf29b3e2fa" in 2.599s (2.599s including waiting). Image size: 218072700 bytes.' metadata: creationTimestamp: "2026-07-13T20:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:57Z" name: switch-graph-cbd18-7b6659b66b-s74jh.18c1f371cf420049 namespace: kserve-ci-e2e-test resourceVersion: "15097" uid: 39bac4b2-0a6e-4cff-a264-143da8eca8ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cbd18} kind: Pod name: switch-graph-cbd18-7b6659b66b-s74jh namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 65a76ec0-4a2a-455f-9c09-638cb6e3986f kind: Event lastTimestamp: "2026-07-13T20:29:57Z" message: 'Created container: switch-graph-cbd18' metadata: creationTimestamp: "2026-07-13T20:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:57Z" name: switch-graph-cbd18-7b6659b66b-s74jh.18c1f371d306d0bd namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: 4101c0a6-6d7c-40cb-9c49-950ae95320b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cbd18} kind: Pod name: switch-graph-cbd18-7b6659b66b-s74jh namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 65a76ec0-4a2a-455f-9c09-638cb6e3986f kind: Event lastTimestamp: "2026-07-13T20:29:57Z" message: Started container switch-graph-cbd18 metadata: creationTimestamp: "2026-07-13T20:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:29:57Z" name: switch-graph-cbd18-7b6659b66b-s74jh.18c1f371d375e42a namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 4da98de2-b258-45a8-a50c-dbafac8f8a22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cbd18} kind: Pod name: switch-graph-cbd18-7b6659b66b-s74jh namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 65a76ec0-4a2a-455f-9c09-638cb6e3986f kind: Event lastTimestamp: "2026-07-13T20:30:04Z" message: Stopping container switch-graph-cbd18 metadata: creationTimestamp: "2026-07-13T20:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:04Z" name: switch-graph-cbd18-7b6659b66b-s74jh.18c1f373531778e6 namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: 004fd917-3896-4648-a460-9465ecd33116 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T20:30:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cbd18} kind: Pod name: switch-graph-cbd18-7b6659b66b-s74jh namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 65a76ec0-4a2a-455f-9c09-638cb6e3986f kind: Event lastTimestamp: "2026-07-13T20:30:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T20:30:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T20:30:33Z" name: switch-graph-cbd18-7b6659b66b-s74jh.18c1f3746487c2eb namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: c8855628-5677-4bb2-b4eb-1bfad24f81f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-97.ec2.internal source: component: kubelet host: ip-10-0-134-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-cbd18-7b6659b66b namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 3504de90-f285-4e52-ae4d-2f289b1ceabd kind: Event lastTimestamp: "2026-07-13T20:29:53Z" message: 'Created pod: switch-graph-cbd18-7b6659b66b-s74jh' metadata: creationTimestamp: "2026-07-13T20:29:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:53Z" name: switch-graph-cbd18-7b6659b66b.18c1f370f5158ff7 namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: be537417-ced7-49ba-9695-15b8936e530c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-cbd18 namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a927158c-c497-4be1-9f4c-2a73d9077c9c kind: Event lastTimestamp: "2026-07-13T20:29:53Z" message: Scaled up replica set switch-graph-cbd18-7b6659b66b from 0 to 1 metadata: creationTimestamp: "2026-07-13T20:29:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T20:29:53Z" name: switch-graph-cbd18.18c1f370f41c9dea namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 8b5a43da-9931-4e4e-b361-68e36cc0a973 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-cbd18 namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: 983a8be1-fe13-4ab1-9855-496d361e23f0 kind: Event lastTimestamp: "2026-07-13T20:29:53Z" message: InferenceGraph [switch-graph-cbd18] is Ready metadata: creationTimestamp: "2026-07-13T20:29:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:29:53Z" name: switch-graph-cbd18.18c1f370fa972856 namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: 97013ece-2c24-46a0-8168-1e8c24cc38d4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-cbd18 namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: 983a8be1-fe13-4ab1-9855-496d361e23f0 kind: Event lastTimestamp: "2026-07-13T20:29:54Z" message: 'Failed to update status for InferenceGraph "switch-graph-cbd18": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-cbd18": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:29:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:29:54Z" name: switch-graph-cbd18.18c1f370fddade46 namespace: kserve-ci-e2e-test resourceVersion: "15061" uid: a9f972ae-0b50-4ffe-9d3a-8eb4667f3c10 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T20:29:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-cbd18 namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: 983a8be1-fe13-4ab1-9855-496d361e23f0 kind: Event lastTimestamp: "2026-07-13T20:29:54Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-cbd18": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T20:29:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T20:29:54Z" name: switch-graph-cbd18.18c1f370fddb268e namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: 50f06ba7-942d-48e8-9c1e-7129f8730c6b reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning kind: EventList metadata: resourceVersion: "33775"