--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:20:41.319440Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3dba1-74bc59c7d7-5c6cv namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: 60e8b2c7-1290-4184-af29-acdde56c15f8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-3dba1-74bc59c7d7-5c6cv to ip-10-0-131-123.ec2.internal metadata: creationTimestamp: "2026-07-10T19:20:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:20:41Z" name: ensemble-graph-3dba1-74bc59c7d7-5c6cv.18c103ec68b7795d namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 2771a075-935b-4818-ae55-28b523c5986b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:41Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3dba1-74bc59c7d7-5c6cv namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 60e8b2c7-1290-4184-af29-acdde56c15f8 kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:20:41Z" name: ensemble-graph-3dba1-74bc59c7d7-5c6cv.18c103ec82342906 namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: bdf8f0fe-df6e-4db5-b733-75c437585e20 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3dba1} kind: Pod name: ensemble-graph-3dba1-74bc59c7d7-5c6cv namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: 60e8b2c7-1290-4184-af29-acdde56c15f8 kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" already present on machine metadata: creationTimestamp: "2026-07-10T19:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:41Z" name: ensemble-graph-3dba1-74bc59c7d7-5c6cv.18c103ec83692710 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: a02dbfa0-fc05-43e4-8798-7614aa6734f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3dba1} kind: Pod name: ensemble-graph-3dba1-74bc59c7d7-5c6cv namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: 60e8b2c7-1290-4184-af29-acdde56c15f8 kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: 'Created container: ensemble-graph-3dba1' metadata: creationTimestamp: "2026-07-10T19:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:41Z" name: ensemble-graph-3dba1-74bc59c7d7-5c6cv.18c103ec864ef00f namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: 826edf01-9d53-44e7-b43a-940e62e1a6ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3dba1} kind: Pod name: ensemble-graph-3dba1-74bc59c7d7-5c6cv namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: 60e8b2c7-1290-4184-af29-acdde56c15f8 kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: Started container ensemble-graph-3dba1 metadata: creationTimestamp: "2026-07-10T19:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:41Z" name: ensemble-graph-3dba1-74bc59c7d7-5c6cv.18c103ec86bb1be9 namespace: kserve-ci-e2e-test resourceVersion: "20857" uid: 858238a3-bc9c-471b-b18d-4d41c06dc4c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3dba1} kind: Pod name: ensemble-graph-3dba1-74bc59c7d7-5c6cv namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: 60e8b2c7-1290-4184-af29-acdde56c15f8 kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: Stopping container ensemble-graph-3dba1 metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:51Z" name: ensemble-graph-3dba1-74bc59c7d7-5c6cv.18c103eec0dd3820 namespace: kserve-ci-e2e-test resourceVersion: "20955" uid: 773c731d-eb27-4f08-a85d-22694f736262 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:20:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3dba1} kind: Pod name: ensemble-graph-3dba1-74bc59c7d7-5c6cv namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: 60e8b2c7-1290-4184-af29-acdde56c15f8 kind: Event lastTimestamp: "2026-07-10T19:21:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:20:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:18Z" name: ensemble-graph-3dba1-74bc59c7d7-5c6cv.18c103ef499901dc namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: f7f24505-40fc-458f-ae82-e0acb4eae3b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-3dba1-74bc59c7d7 namespace: kserve-ci-e2e-test resourceVersion: "20826" uid: c2d7875d-ae0a-44c1-b17a-1ea8d174697f kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: 'Created pod: ensemble-graph-3dba1-74bc59c7d7-5c6cv' metadata: creationTimestamp: "2026-07-10T19:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:41Z" name: ensemble-graph-3dba1-74bc59c7d7.18c103ec6823e4f5 namespace: kserve-ci-e2e-test resourceVersion: "20831" uid: a72c203f-a210-4550-ba39-79832b20a184 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-3dba1 namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: 92523e65-dbeb-49f9-b63a-a0040569888a kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: Scaled up replica set ensemble-graph-3dba1-74bc59c7d7 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:41Z" name: ensemble-graph-3dba1.18c103ec676eccca namespace: kserve-ci-e2e-test resourceVersion: "20827" uid: d203174f-41a1-40f3-936c-db44ef672668 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-3dba1 namespace: kserve-ci-e2e-test resourceVersion: "20843" uid: 4ce487e5-d96a-407b-b7a3-98b920bac39e kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: InferenceGraph [ensemble-graph-3dba1] is Ready metadata: creationTimestamp: "2026-07-10T19:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:41Z" name: ensemble-graph-3dba1.18c103ec6b1b55f5 namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 3919ffaa-e40f-4b62-b012-0e066f6bfc6b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-3dba1 namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 4ce487e5-d96a-407b-b7a3-98b920bac39e kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-3dba1": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-3dba1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:41Z" name: ensemble-graph-3dba1.18c103ec6d4f4bda namespace: kserve-ci-e2e-test resourceVersion: "20849" uid: b611f7ed-3ba9-4821-82eb-08134f3cb680 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-3dba1 namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 4ce487e5-d96a-407b-b7a3-98b920bac39e kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-3dba1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:41Z" name: ensemble-graph-3dba1.18c103ec6d4f918d namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: c3df8f77-7cbc-4e2c-b415-a7849d63298a reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:22:01.637110Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d232d-596f769657-56w4v namespace: kserve-ci-e2e-test resourceVersion: "21883" uid: 231cd065-32fe-423a-8b03-c900b2721e41 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-d232d-596f769657-56w4v to ip-10-0-131-123.ec2.internal metadata: creationTimestamp: "2026-07-10T19:22:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:22:01Z" name: ensemble-graph-d232d-596f769657-56w4v.18c103ff1c05e224 namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 649465c5-8a01-408c-905e-ffce2ff09dbc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:01Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d232d-596f769657-56w4v namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: 231cd065-32fe-423a-8b03-c900b2721e41 kind: Event lastTimestamp: "2026-07-10T19:22:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-d232d-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:01Z" name: ensemble-graph-d232d-596f769657-56w4v.18c103ff25540200 namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: 10d15be2-8fd5-4a57-9c73-8ebd8d175a39 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:02Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d232d-596f769657-56w4v namespace: kserve-ci-e2e-test resourceVersion: "21892" uid: 231cd065-32fe-423a-8b03-c900b2721e41 kind: Event lastTimestamp: "2026-07-10T19:22:02Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:22:02Z" name: ensemble-graph-d232d-596f769657-56w4v.18c103ff58fa7383 namespace: kserve-ci-e2e-test resourceVersion: "21905" uid: ef235ebf-47cc-46b6-bca0-f06a7b3218e6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d232d} kind: Pod name: ensemble-graph-d232d-596f769657-56w4v namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: 231cd065-32fe-423a-8b03-c900b2721e41 kind: Event lastTimestamp: "2026-07-10T19:22:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" already present on machine metadata: creationTimestamp: "2026-07-10T19:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:02Z" name: ensemble-graph-d232d-596f769657-56w4v.18c103ff5a395e71 namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: ad13a944-f6bb-4f66-9ade-29de4d9c001e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d232d} kind: Pod name: ensemble-graph-d232d-596f769657-56w4v namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: 231cd065-32fe-423a-8b03-c900b2721e41 kind: Event lastTimestamp: "2026-07-10T19:22:02Z" message: 'Created container: ensemble-graph-d232d' metadata: creationTimestamp: "2026-07-10T19:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:02Z" name: ensemble-graph-d232d-596f769657-56w4v.18c103ff5cdc28b6 namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: 4ae9392c-7daf-4b23-840d-2ff65a1238c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d232d} kind: Pod name: ensemble-graph-d232d-596f769657-56w4v namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: 231cd065-32fe-423a-8b03-c900b2721e41 kind: Event lastTimestamp: "2026-07-10T19:22:02Z" message: Started container ensemble-graph-d232d metadata: creationTimestamp: "2026-07-10T19:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:02Z" name: ensemble-graph-d232d-596f769657-56w4v.18c103ff5d4a98b8 namespace: kserve-ci-e2e-test resourceVersion: "21911" uid: dbffff00-f4fb-4bd8-8cf2-72dd5d82f7ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d232d} kind: Pod name: ensemble-graph-d232d-596f769657-56w4v namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: 231cd065-32fe-423a-8b03-c900b2721e41 kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: Stopping container ensemble-graph-d232d metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:16Z" name: ensemble-graph-d232d-596f769657-56w4v.18c10472482a7571 namespace: kserve-ci-e2e-test resourceVersion: "25678" uid: 94fb1d02-d783-4827-8edf-962b03b04097 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:30:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d232d} kind: Pod name: ensemble-graph-d232d-596f769657-56w4v namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: 231cd065-32fe-423a-8b03-c900b2721e41 kind: Event lastTimestamp: "2026-07-10T19:30:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:30:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:44Z" name: ensemble-graph-d232d-596f769657-56w4v.18c10472eadc67ce namespace: kserve-ci-e2e-test resourceVersion: "26162" uid: 867910de-6f39-4400-a7b4-5797176bc05a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-d232d-596f769657 namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: 0225647a-3cdc-417c-8f58-0fcde892f723 kind: Event lastTimestamp: "2026-07-10T19:22:01Z" message: 'Created pod: ensemble-graph-d232d-596f769657-56w4v' metadata: creationTimestamp: "2026-07-10T19:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:01Z" name: ensemble-graph-d232d-596f769657.18c103ff1b999885 namespace: kserve-ci-e2e-test resourceVersion: "21884" uid: 89f4872f-57af-46f8-a50e-73b654c0ad71 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-d232d namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: c36a1106-b8a2-4673-a39f-996110331c9d kind: Event lastTimestamp: "2026-07-10T19:22:01Z" message: Scaled up replica set ensemble-graph-d232d-596f769657 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:01Z" name: ensemble-graph-d232d.18c103ff1adffe40 namespace: kserve-ci-e2e-test resourceVersion: "21881" uid: 621c37cb-0d4e-4da5-bd69-4308afad60db reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-d232d namespace: kserve-ci-e2e-test resourceVersion: "21895" uid: 3c8bfc15-b75f-455d-821d-4d8d1d070059 kind: Event lastTimestamp: "2026-07-10T19:22:01Z" message: InferenceGraph [ensemble-graph-d232d] is Ready metadata: creationTimestamp: "2026-07-10T19:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:22:01Z" name: ensemble-graph-d232d.18c103ff1ff5d371 namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: bffbfa29-971f-4a91-838e-401eaaf4e8fd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d232d namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: dba68af9-0ffa-4867-9449-c153397379bd kind: Event lastTimestamp: "2026-07-10T19:22:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:16Z" name: ensemble-graph-d232d.18c10402996e0fe1 namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 78f79aae-98d7-4287-9d28-033a91fc27ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d232d namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: dba68af9-0ffa-4867-9449-c153397379bd kind: Event lastTimestamp: "2026-07-10T19:22:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:16Z" name: ensemble-graph-d232d.18c1040299ba3cb9 namespace: kserve-ci-e2e-test resourceVersion: "22034" uid: 1f32333f-d679-4343-b05a-51b36e4a7f7e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d232d namespace: kserve-ci-e2e-test resourceVersion: "22033" uid: dba68af9-0ffa-4867-9449-c153397379bd kind: Event lastTimestamp: "2026-07-10T19:22:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:31Z" name: ensemble-graph-d232d.18c10406186e8ec7 namespace: kserve-ci-e2e-test resourceVersion: "22191" uid: 586d7eb1-de34-41c6-b50b-f15a20c4fda1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-d232d namespace: kserve-ci-e2e-test resourceVersion: "22033" uid: dba68af9-0ffa-4867-9449-c153397379bd kind: Event lastTimestamp: "2026-07-10T19:22:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:31Z" name: ensemble-graph-d232d.18c1040618d2eb46 namespace: kserve-ci-e2e-test resourceVersion: "22194" uid: d215a511-62a7-4733-b38f-30463845aaf2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:30:42.680118Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1c787-predictor-794854c758-r7j46 namespace: kserve-ci-e2e-test resourceVersion: "26113" uid: d43689cc-70b7-41fd-9e17-a1f9971ae6ee kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1c787-predictor-794854c758-r7j46 to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:30:42Z" name: error-404-isvc-1c787-predictor-794854c758-r7j46.18c104786c9b4a0f namespace: kserve-ci-e2e-test resourceVersion: "26117" uid: 8d1f5c96-69ce-41c1-9ff8-39e1ef496c24 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1c787-predictor-794854c758-r7j46 namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: d43689cc-70b7-41fd-9e17-a1f9971ae6ee kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1c787-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:42Z" name: error-404-isvc-1c787-predictor-794854c758-r7j46.18c1047877eaba0b namespace: kserve-ci-e2e-test resourceVersion: "26132" uid: 9ae282e7-6ccf-47ff-900a-c4be818cd5cb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1c787-predictor-794854c758-r7j46 namespace: kserve-ci-e2e-test resourceVersion: "26124" uid: d43689cc-70b7-41fd-9e17-a1f9971ae6ee kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:30:43Z" name: error-404-isvc-1c787-predictor-794854c758-r7j46.18c10478aa4c2e77 namespace: kserve-ci-e2e-test resourceVersion: "26149" uid: 0ed35e51-b69f-4c37-8e1d-78aec392cd68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1c787-predictor-794854c758-r7j46 namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: d43689cc-70b7-41fd-9e17-a1f9971ae6ee kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:43Z" name: error-404-isvc-1c787-predictor-794854c758-r7j46.18c10478abae12d2 namespace: kserve-ci-e2e-test resourceVersion: "26151" uid: a64ce0d4-6dcb-4604-89d4-18b1f34f554c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1c787-predictor-794854c758-r7j46 namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: d43689cc-70b7-41fd-9e17-a1f9971ae6ee kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:43Z" name: error-404-isvc-1c787-predictor-794854c758-r7j46.18c10478ae6e7433 namespace: kserve-ci-e2e-test resourceVersion: "26153" uid: 5f721cb9-e8d2-4ee3-ac59-9872d8ebae6d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1c787-predictor-794854c758-r7j46 namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: d43689cc-70b7-41fd-9e17-a1f9971ae6ee kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:43Z" name: error-404-isvc-1c787-predictor-794854c758-r7j46.18c10478aee4141d namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: e64f0386-d450-4ad3-b2ce-2ad19961833f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1c787-predictor-794854c758-r7j46 namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: d43689cc-70b7-41fd-9e17-a1f9971ae6ee kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:43Z" name: error-404-isvc-1c787-predictor-794854c758-r7j46.18c10478aeecccc1 namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: 5eff1905-c8f9-4893-a183-7d708c44fa74 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1c787-predictor-794854c758-r7j46 namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: d43689cc-70b7-41fd-9e17-a1f9971ae6ee kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:43Z" name: error-404-isvc-1c787-predictor-794854c758-r7j46.18c10478b23e2874 namespace: kserve-ci-e2e-test resourceVersion: "26159" uid: 8c203b28-3166-476e-b0b5-2de958082d43 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1c787-predictor-794854c758-r7j46 namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: d43689cc-70b7-41fd-9e17-a1f9971ae6ee kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:43Z" name: error-404-isvc-1c787-predictor-794854c758-r7j46.18c10478b2b3c019 namespace: kserve-ci-e2e-test resourceVersion: "26160" uid: fd346769-e036-4a41-a4d9-10cec212b02b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-10T19:30:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1c787-predictor-794854c758-r7j46 namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: d43689cc-70b7-41fd-9e17-a1f9971ae6ee kind: Event lastTimestamp: "2026-07-10T19:31:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:30:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:30Z" name: error-404-isvc-1c787-predictor-794854c758-r7j46.18c10478ea631ac6 namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: 4fc3f552-11fd-46fd-81a5-d373a367ac34 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1c787-predictor-794854c758-r7j46 namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: d43689cc-70b7-41fd-9e17-a1f9971ae6ee kind: Event lastTimestamp: "2026-07-10T19:48:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:48:01Z" name: error-404-isvc-1c787-predictor-794854c758-r7j46.18c1056a6845f17d namespace: kserve-ci-e2e-test resourceVersion: "33842" uid: 08b04a03-2c78-4dd3-b1fb-41ee5ed5b472 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1c787-predictor-794854c758-r7j46 namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: d43689cc-70b7-41fd-9e17-a1f9971ae6ee kind: Event lastTimestamp: "2026-07-10T19:48:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:48:01Z" name: error-404-isvc-1c787-predictor-794854c758-r7j46.18c1056a68470d69 namespace: kserve-ci-e2e-test resourceVersion: "33843" uid: 3074a85f-514b-4395-9bad-6f905d31a308 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1c787-predictor-794854c758 namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: d5dcc19a-4bbf-4b3a-8765-b78e80934581 kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: 'Created pod: error-404-isvc-1c787-predictor-794854c758-r7j46' metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:30:42Z" name: error-404-isvc-1c787-predictor-794854c758.18c104786bf562bd namespace: kserve-ci-e2e-test resourceVersion: "26114" uid: 5393269b-5c01-4639-aee5-de608d367790 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1c787-predictor namespace: kserve-ci-e2e-test resourceVersion: "26104" uid: 52d69452-c45f-4068-b553-eb989da4d892 kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: Scaled up replica set error-404-isvc-1c787-predictor-794854c758 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:30:42Z" name: error-404-isvc-1c787-predictor.18c10478699f5bd9 namespace: kserve-ci-e2e-test resourceVersion: "26107" uid: 3d0bb72f-84e5-447f-820a-28d45a620976 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:30:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1c787-predictor namespace: kserve-ci-e2e-test resourceVersion: "26105" uid: 3e46f715-9e9a-4bdd-a9e0-bfeb5883a478 kind: Event lastTimestamp: "2026-07-10T19:31:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:57Z" name: error-404-isvc-1c787-predictor.18c1047be80caf86 namespace: kserve-ci-e2e-test resourceVersion: "27040" uid: 305533fc-80e8-4c6f-bbfb-698016025ffc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:30:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1c787-predictor namespace: kserve-ci-e2e-test resourceVersion: "26105" uid: 3e46f715-9e9a-4bdd-a9e0-bfeb5883a478 kind: Event lastTimestamp: "2026-07-10T19:31:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:57Z" name: error-404-isvc-1c787-predictor.18c1047be873a231 namespace: kserve-ci-e2e-test resourceVersion: "27041" uid: 90542bf2-bea5-435a-89ff-d4f460a9ec92 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1c787-predictor namespace: kserve-ci-e2e-test resourceVersion: "26291" uid: 3e46f715-9e9a-4bdd-a9e0-bfeb5883a478 kind: Event lastTimestamp: "2026-07-10T19:31:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:12Z" name: error-404-isvc-1c787-predictor.18c1047f672b58fa namespace: kserve-ci-e2e-test resourceVersion: "26428" uid: 4df64744-7ac1-4ae1-ba34-0411217a6878 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1c787-predictor namespace: kserve-ci-e2e-test resourceVersion: "26291" uid: 3e46f715-9e9a-4bdd-a9e0-bfeb5883a478 kind: Event lastTimestamp: "2026-07-10T19:31:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:12Z" name: error-404-isvc-1c787-predictor.18c1047f67987d51 namespace: kserve-ci-e2e-test resourceVersion: "26430" uid: 9da58624-4cc6-4df5-981d-7080818ab0b5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1c787 namespace: kserve-ci-e2e-test resourceVersion: "26088" uid: 9e4e1c03-c35f-48fc-bf18-54130c4c4909 kind: Event lastTimestamp: "2026-07-10T19:31:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:40Z" name: error-404-isvc-1c787.18c1047869b8831d namespace: kserve-ci-e2e-test resourceVersion: "26850" uid: 0dab050a-8c46-4948-8c24-8437e5b697c2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1c787 namespace: kserve-ci-e2e-test resourceVersion: "26088" uid: 9e4e1c03-c35f-48fc-bf18-54130c4c4909 kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: 'Failed to update status for InferenceService "error-404-isvc-1c787": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1c787": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:30:42Z" name: error-404-isvc-1c787.18c104786a37be51 namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: 9529edb8-4004-40a1-be0f-37cd01b7d61e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1c787 namespace: kserve-ci-e2e-test resourceVersion: "26088" uid: 9e4e1c03-c35f-48fc-bf18-54130c4c4909 kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1c787": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:30:42Z" name: error-404-isvc-1c787.18c104786a37e625 namespace: kserve-ci-e2e-test resourceVersion: "26111" uid: c3e7fd47-ce6d-4444-8eca-3bd914cac6c7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1c787 namespace: kserve-ci-e2e-test resourceVersion: "26847" uid: 9e4e1c03-c35f-48fc-bf18-54130c4c4909 kind: Event lastTimestamp: "2026-07-10T19:31:40Z" message: InferenceService [error-404-isvc-1c787] is Ready metadata: creationTimestamp: "2026-07-10T19:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:40Z" name: error-404-isvc-1c787.18c10485f9533998 namespace: kserve-ci-e2e-test resourceVersion: "26849" uid: cceb11c5-2189-4688-8a94-6f752b411c26 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:31:26.953157Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: b86ae9c0-0709-410b-82ee-42846b74d896 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1e1e7-predictor-98b49b564-fwt8l to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:31:26Z" name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l.18c10482bb7c43e0 namespace: kserve-ci-e2e-test resourceVersion: "26675" uid: ccb25982-a084-4f2a-bd20-fb23e5a064e2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: b86ae9c0-0709-410b-82ee-42846b74d896 kind: Event lastTimestamp: "2026-07-10T19:31:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1e1e7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:27Z" name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l.18c10482cac08f2f namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: a5182564-be73-4380-9695-459b1a09f43a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: b86ae9c0-0709-410b-82ee-42846b74d896 kind: Event lastTimestamp: "2026-07-10T19:31:27Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:31:27Z" name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l.18c10482f9b8ae8e namespace: kserve-ci-e2e-test resourceVersion: "26717" uid: 293325bf-53c0-490f-b74e-9e2253e0ae52 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: b86ae9c0-0709-410b-82ee-42846b74d896 kind: Event lastTimestamp: "2026-07-10T19:31:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:28Z" name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l.18c10482fb47450f namespace: kserve-ci-e2e-test resourceVersion: "26719" uid: 44cdeee5-4a70-4cf5-9f58-a60e2a0b5fa0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: b86ae9c0-0709-410b-82ee-42846b74d896 kind: Event lastTimestamp: "2026-07-10T19:31:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:28Z" name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l.18c10482fe11b68b namespace: kserve-ci-e2e-test resourceVersion: "26723" uid: a87700d5-c2df-4a38-981e-6a0079f39151 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: b86ae9c0-0709-410b-82ee-42846b74d896 kind: Event lastTimestamp: "2026-07-10T19:31:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:28Z" name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l.18c10482fe82243e namespace: kserve-ci-e2e-test resourceVersion: "26724" uid: 096a5da9-daad-4401-8dde-d76e9ad58512 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: b86ae9c0-0709-410b-82ee-42846b74d896 kind: Event lastTimestamp: "2026-07-10T19:31:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:28Z" name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l.18c10482fe8b9581 namespace: kserve-ci-e2e-test resourceVersion: "26725" uid: 17f5907a-6875-4743-bcb2-b695b63edbf3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: b86ae9c0-0709-410b-82ee-42846b74d896 kind: Event lastTimestamp: "2026-07-10T19:31:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:28Z" name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l.18c10483025a8330 namespace: kserve-ci-e2e-test resourceVersion: "26726" uid: b772ee30-6d9e-4b61-bfbc-a7ad43772450 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: b86ae9c0-0709-410b-82ee-42846b74d896 kind: Event lastTimestamp: "2026-07-10T19:31:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:28Z" name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l.18c1048302ce9d43 namespace: kserve-ci-e2e-test resourceVersion: "26727" uid: 3adc572d-9987-4958-9494-1e8b30ce9d05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: b86ae9c0-0709-410b-82ee-42846b74d896 kind: Event lastTimestamp: "2026-07-10T19:32:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:32:04Z" name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l.18c10483344c78f5 namespace: kserve-ci-e2e-test resourceVersion: "27089" uid: 0fed07a5-de43-4ac7-8e58-57b9f22b7897 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:40:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: b86ae9c0-0709-410b-82ee-42846b74d896 kind: Event lastTimestamp: "2026-07-10T19:40:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:40:41Z" name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l.18c10503e1613a75 namespace: kserve-ci-e2e-test resourceVersion: "30779" uid: a7361816-cb25-4635-a334-949f7fe649b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:40:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: b86ae9c0-0709-410b-82ee-42846b74d896 kind: Event lastTimestamp: "2026-07-10T19:40:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:40:41Z" name: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l.18c10503e161e853 namespace: kserve-ci-e2e-test resourceVersion: "30781" uid: df0e45b5-e936-4cb4-8a57-1d5e34546792 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1e1e7-predictor-98b49b564 namespace: kserve-ci-e2e-test resourceVersion: "26664" uid: 7a483726-ec19-49fe-ac7e-eae6a6d0a623 kind: Event lastTimestamp: "2026-07-10T19:31:26Z" message: 'Created pod: error-404-isvc-1e1e7-predictor-98b49b564-fwt8l' metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:26Z" name: error-404-isvc-1e1e7-predictor-98b49b564.18c10482bade2422 namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: 89d7f963-9ad4-49c6-b320-1ade923848ed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1e1e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "26662" uid: e47b49ec-3278-4a19-bc52-e1457c233abb kind: Event lastTimestamp: "2026-07-10T19:31:26Z" message: Scaled up replica set error-404-isvc-1e1e7-predictor-98b49b564 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:26Z" name: error-404-isvc-1e1e7-predictor.18c10482b7f2d0ad namespace: kserve-ci-e2e-test resourceVersion: "26665" uid: 7a6a21f8-1261-4754-b04c-3dbfc674735a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1e1e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: 9895e4b1-5b14-47ef-9bba-749f3de45051 kind: Event lastTimestamp: "2026-07-10T19:32:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:11Z" name: error-404-isvc-1e1e7-predictor.18c1048636622e19 namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: a341228a-6d53-4001-9bf6-74e1fc87a480 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1e1e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: 9895e4b1-5b14-47ef-9bba-749f3de45051 kind: Event lastTimestamp: "2026-07-10T19:32:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:11Z" name: error-404-isvc-1e1e7-predictor.18c1048636c5dfa8 namespace: kserve-ci-e2e-test resourceVersion: "27139" uid: 2a5a5988-7b31-46a6-9411-3b7a3d3f3c4a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:31:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1e1e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: 9895e4b1-5b14-47ef-9bba-749f3de45051 kind: Event lastTimestamp: "2026-07-10T19:32:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:31:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:41Z" name: error-404-isvc-1e1e7-predictor.18c10489b56e4723 namespace: kserve-ci-e2e-test resourceVersion: "27422" uid: b59e356d-aec1-4f13-a398-151261aa8bb1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:31:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1e1e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: 9895e4b1-5b14-47ef-9bba-749f3de45051 kind: Event lastTimestamp: "2026-07-10T19:32:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:31:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:41Z" name: error-404-isvc-1e1e7-predictor.18c10489b5d184f2 namespace: kserve-ci-e2e-test resourceVersion: "27423" uid: 4d54619e-711d-4d3c-8c87-5c3d322f9778 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: 6f9f5dab-5a16-4aa5-a6e7-5f1553b89256 kind: Event lastTimestamp: "2026-07-10T19:32:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:32:15Z" name: error-404-isvc-1e1e7.18c10482b7fa0d4f namespace: kserve-ci-e2e-test resourceVersion: "27186" uid: 05210a6b-12ad-4379-aa60-9ba4117d016a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: 6f9f5dab-5a16-4aa5-a6e7-5f1553b89256 kind: Event lastTimestamp: "2026-07-10T19:31:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-1e1e7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1e1e7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:26Z" name: error-404-isvc-1e1e7.18c10482b8678b8b namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: c5badc2b-04ca-4d49-af47-54b0f4ce7b2e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: 6f9f5dab-5a16-4aa5-a6e7-5f1553b89256 kind: Event lastTimestamp: "2026-07-10T19:31:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1e1e7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:26Z" name: error-404-isvc-1e1e7.18c10482b867caff namespace: kserve-ci-e2e-test resourceVersion: "26669" uid: 65c7ec11-bd6e-4cc7-9c29-1aa70d23a554 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "27183" uid: 6f9f5dab-5a16-4aa5-a6e7-5f1553b89256 kind: Event lastTimestamp: "2026-07-10T19:32:15Z" message: InferenceService [error-404-isvc-1e1e7] is Ready metadata: creationTimestamp: "2026-07-10T19:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:32:15Z" name: error-404-isvc-1e1e7.18c1048df404b631 namespace: kserve-ci-e2e-test resourceVersion: "27184" uid: 82a7ef48-cba4-44d1-9ea0-ccf6988ed15f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:11:02.441369Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: fb9cfa8c-7f6a-4a51-80a8-5fe6fee09f23 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:11:02Z" name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r.18c10365a0e514e5 namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: 35a29a8e-f8b3-426b-a16b-8874ab88970d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: fb9cfa8c-7f6a-4a51-80a8-5fe6fee09f23 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:11:02Z" name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r.18c10365baeef5f2 namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 7a5c4b20-37c0-41ce-a2d9-89137b8b032b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: fb9cfa8c-7f6a-4a51-80a8-5fe6fee09f23 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r.18c10365bc97f57a namespace: kserve-ci-e2e-test resourceVersion: "16283" uid: a61fd029-35ee-41fc-adcd-c04b7551345f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: fb9cfa8c-7f6a-4a51-80a8-5fe6fee09f23 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r.18c10365bf4814d7 namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: b7d3889a-5b01-4ce9-8dfe-0c0645bfebb9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: fb9cfa8c-7f6a-4a51-80a8-5fe6fee09f23 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r.18c10365bfb8fd98 namespace: kserve-ci-e2e-test resourceVersion: "16285" uid: 01781fd8-0add-49bc-827e-d4235c4469cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: fb9cfa8c-7f6a-4a51-80a8-5fe6fee09f23 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r.18c10365bfc3a6c7 namespace: kserve-ci-e2e-test resourceVersion: "16286" uid: 62f1c7c3-6376-4c16-87c1-679cc4433f05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: fb9cfa8c-7f6a-4a51-80a8-5fe6fee09f23 kind: Event lastTimestamp: "2026-07-10T19:11:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:11:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:03Z" name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r.18c10365c2ac4a51 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 50642398-5c9c-4a72-9567-414f7ef527a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: fb9cfa8c-7f6a-4a51-80a8-5fe6fee09f23 kind: Event lastTimestamp: "2026-07-10T19:11:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:11:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:03Z" name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r.18c10365c3311b4c namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: e1e5e38b-18d0-47d0-9b89-48f9e50360c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-10T19:11:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: fb9cfa8c-7f6a-4a51-80a8-5fe6fee09f23 kind: Event lastTimestamp: "2026-07-10T19:20:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:11:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:19Z" name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r.18c10365cd9e45e8 namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: ff97fb1f-854e-427c-ae9f-818e0b9f01aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: fb9cfa8c-7f6a-4a51-80a8-5fe6fee09f23 kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:17Z" name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r.18c103e6d31c7a6a namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 57b615de-807e-44fd-b7e1-ae216d6c07b7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: fb9cfa8c-7f6a-4a51-80a8-5fe6fee09f23 kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:17Z" name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r.18c103e6d31d2692 namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 54d9362e-9bd5-40ae-97dc-127f0bc592f6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: fb9cfa8c-7f6a-4a51-80a8-5fe6fee09f23 kind: Event lastTimestamp: "2026-07-10T19:20:19Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:20:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:19Z" name: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r.18c103e741fe1467 namespace: kserve-ci-e2e-test resourceVersion: "20606" uid: f0ed71e4-f244-43d2-8d73-6f038fae55bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-31f26-predictor-c7f49fbf8 namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: 07429d8a-d02f-4649-a12c-5bffe90c41fe kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: 'Created pod: error-404-isvc-31f26-predictor-c7f49fbf8-gxg8r' metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:02Z" name: error-404-isvc-31f26-predictor-c7f49fbf8.18c10365a015c8e9 namespace: kserve-ci-e2e-test resourceVersion: "16249" uid: d90d1b63-a02f-4e4b-8db2-8225bb315a57 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-31f26-predictor namespace: kserve-ci-e2e-test resourceVersion: "16238" uid: 61208d86-bba9-4147-b606-69bde29b2e40 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Scaled up replica set error-404-isvc-31f26-predictor-c7f49fbf8 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:02Z" name: error-404-isvc-31f26-predictor.18c103659e6021ee namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: 798631b6-07b4-4c74-87d6-bba96507be97 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-31f26-predictor namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: 25abcc74-bd9e-4cae-8ff9-c2292e6adc39 kind: Event lastTimestamp: "2026-07-10T19:11:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:17Z" name: error-404-isvc-31f26-predictor.18c103691ce39e05 namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: bb2f2fc2-c938-404a-bbc2-31b8c49d5b0c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-31f26-predictor namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: 25abcc74-bd9e-4cae-8ff9-c2292e6adc39 kind: Event lastTimestamp: "2026-07-10T19:11:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:17Z" name: error-404-isvc-31f26-predictor.18c103691d49a842 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: c68ae46b-86f8-4a52-a1b8-4fb5ae5fb8d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:11:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-31f26-predictor namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 25abcc74-bd9e-4cae-8ff9-c2292e6adc39 kind: Event lastTimestamp: "2026-07-10T19:12:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:17Z" name: error-404-isvc-31f26-predictor.18c1036c9be896cb namespace: kserve-ci-e2e-test resourceVersion: "17011" uid: c9d59817-c4df-4202-88ee-1ead4227e912 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:11:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-31f26-predictor namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 25abcc74-bd9e-4cae-8ff9-c2292e6adc39 kind: Event lastTimestamp: "2026-07-10T19:12:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:17Z" name: error-404-isvc-31f26-predictor.18c1036c9c58c7ef namespace: kserve-ci-e2e-test resourceVersion: "17012" uid: 10958bad-d105-4a34-bf86-cf1bfdb653f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: 07a6f7cd-6e42-4c96-84c1-842966a223f5 kind: Event lastTimestamp: "2026-07-10T19:11:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:49Z" name: error-404-isvc-31f26.18c103659e5e2e65 namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: 47c87000-a7e9-4429-8056-4adf931c5163 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: 07a6f7cd-6e42-4c96-84c1-842966a223f5 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: 'Failed to update status for InferenceService "error-404-isvc-31f26": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-31f26": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:02Z" name: error-404-isvc-31f26.18c103659f049abb namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: 71dfab1c-41dd-4984-afaa-9c97bb94d29e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: 07a6f7cd-6e42-4c96-84c1-842966a223f5 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-31f26": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:02Z" name: error-404-isvc-31f26.18c103659f04cc71 namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: 1ce60750-ef6e-4e40-a6fb-996f83034b2a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: 07a6f7cd-6e42-4c96-84c1-842966a223f5 kind: Event lastTimestamp: "2026-07-10T19:11:49Z" message: InferenceService [error-404-isvc-31f26] is Ready metadata: creationTimestamp: "2026-07-10T19:11:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:49Z" name: error-404-isvc-31f26.18c1037088e9e16e namespace: kserve-ci-e2e-test resourceVersion: "16745" uid: b1b2f853-cbcb-4c2c-a5fa-84e109d929de reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:19:41.329894Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2 namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: 4973cc2d-3c76-450c-887f-dc31c7e95859 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2 to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:19:41Z" name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2.18c103de710f9f4f namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: b459bcc1-62a8-4356-ac18-0837d04ee956 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 4973cc2d-3c76-450c-887f-dc31c7e95859 kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-3dba1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:41Z" name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2.18c103de77ef1c85 namespace: kserve-ci-e2e-test resourceVersion: "20142" uid: 54bd3f9c-a76e-4bd5-8a65-29d7e16f15ff reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2 namespace: kserve-ci-e2e-test resourceVersion: "20138" uid: 4973cc2d-3c76-450c-887f-dc31c7e95859 kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:19:42Z" name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2.18c103deaf551ecf namespace: kserve-ci-e2e-test resourceVersion: "20162" uid: f95cc821-d96c-41ab-9066-595050c4ce81 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 4973cc2d-3c76-450c-887f-dc31c7e95859 kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:42Z" name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2.18c103deb0b400bb namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: dd955b27-7f0d-4674-908a-2eb27d07090f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 4973cc2d-3c76-450c-887f-dc31c7e95859 kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:42Z" name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2.18c103deb45a485e namespace: kserve-ci-e2e-test resourceVersion: "20170" uid: ea721b5c-f6ed-4d5e-afe3-fbc3cba17d19 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 4973cc2d-3c76-450c-887f-dc31c7e95859 kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:42Z" name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2.18c103deb4d9cf7d namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: 8f35f094-4e95-49c6-929d-450558e6209e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 4973cc2d-3c76-450c-887f-dc31c7e95859 kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:42Z" name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2.18c103deb4e39c52 namespace: kserve-ci-e2e-test resourceVersion: "20174" uid: f733af2d-4f3a-414d-ae5b-5ffee6c7bd68 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 4973cc2d-3c76-450c-887f-dc31c7e95859 kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:42Z" name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2.18c103deb931b20f namespace: kserve-ci-e2e-test resourceVersion: "20175" uid: bf898f02-3a1b-4d56-a616-458509eea273 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 4973cc2d-3c76-450c-887f-dc31c7e95859 kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:42Z" name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2.18c103deb9a87534 namespace: kserve-ci-e2e-test resourceVersion: "20176" uid: 93e7ff17-54d8-40e4-9178-aa9baf2684dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:19:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 4973cc2d-3c76-450c-887f-dc31c7e95859 kind: Event lastTimestamp: "2026-07-10T19:20:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:19:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:20Z" name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2.18c103df199f6f97 namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 2d67ddd4-50d8-4023-9f14-da70521028e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 4973cc2d-3c76-450c-887f-dc31c7e95859 kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:51Z" name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2.18c103eec5450566 namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: 02fe7636-6c9b-423d-8d9f-f1b261aa593d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 4973cc2d-3c76-450c-887f-dc31c7e95859 kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:51Z" name: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2.18c103eec545956c namespace: kserve-ci-e2e-test resourceVersion: "20972" uid: d1cea1cd-6541-420e-9148-5cbcba6c6ce4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3dba1-predictor-6ccc87694d namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 1cfcfc35-f342-4d1a-8f4d-34e22523fcd3 kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: 'Created pod: error-404-isvc-3dba1-predictor-6ccc87694d-zk5s2' metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:19:41Z" name: error-404-isvc-3dba1-predictor-6ccc87694d.18c103de709d601a namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 9480ad22-1484-4fcd-b9a5-9e40b0971208 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3dba1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: 944c1d99-be2b-4127-8ba5-0e9b60251ae1 kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: Scaled up replica set error-404-isvc-3dba1-predictor-6ccc87694d from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:19:41Z" name: error-404-isvc-3dba1-predictor.18c103de6ec8dc87 namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 93cc7b47-ea5e-4cf6-a2e0-9e17a8ff47f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:19:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3dba1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: dd690c53-7b0c-4a67-b0bc-79b443cfc57c kind: Event lastTimestamp: "2026-07-10T19:20:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:26Z" name: error-404-isvc-3dba1-predictor.18c103e1ed545a00 namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 755019e8-ff2d-4d62-98eb-491543caa295 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:19:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3dba1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: dd690c53-7b0c-4a67-b0bc-79b443cfc57c kind: Event lastTimestamp: "2026-07-10T19:20:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:26Z" name: error-404-isvc-3dba1-predictor.18c103e1edc42cd7 namespace: kserve-ci-e2e-test resourceVersion: "20676" uid: e7aeb9b7-52c9-4e42-b777-cbc8e25467d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:20:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3dba1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: dd690c53-7b0c-4a67-b0bc-79b443cfc57c kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:41Z" name: error-404-isvc-3dba1-predictor.18c103e56c73fdff namespace: kserve-ci-e2e-test resourceVersion: "20842" uid: eed07f7e-055a-48a6-a4aa-01de390a308a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:20:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3dba1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: dd690c53-7b0c-4a67-b0bc-79b443cfc57c kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:41Z" name: error-404-isvc-3dba1-predictor.18c103e56cdba572 namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: 5111b220-461e-4c20-bdd5-a5e9d9764c52 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3dba1 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: acb5c44f-2405-4541-a13c-b343bf6d6e21 kind: Event lastTimestamp: "2026-07-10T19:20:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:30Z" name: error-404-isvc-3dba1.18c103de6ef99981 namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: 240f2c74-f8c0-4124-a662-4dee78b34a79 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3dba1 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: acb5c44f-2405-4541-a13c-b343bf6d6e21 kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-3dba1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3dba1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:19:41Z" name: error-404-isvc-3dba1.18c103de6f6c3676 namespace: kserve-ci-e2e-test resourceVersion: "20125" uid: 408a5986-f30b-4652-becc-29a9d20509e0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3dba1 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: acb5c44f-2405-4541-a13c-b343bf6d6e21 kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3dba1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:19:41Z" name: error-404-isvc-3dba1.18c103de6f6c83f1 namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: d1ac6efa-98b7-4e11-a078-8cc2a98fb114 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3dba1 namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: acb5c44f-2405-4541-a13c-b343bf6d6e21 kind: Event lastTimestamp: "2026-07-10T19:20:30Z" message: InferenceService [error-404-isvc-3dba1] is Ready metadata: creationTimestamp: "2026-07-10T19:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:30Z" name: error-404-isvc-3dba1.18c103e9d4998fc4 namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: 13a3c045-7843-4a60-9852-cdf37462fe98 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:10:26.416754Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 69e78f05-d420-4352-bb49-9336e87fa5e8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:10:26Z" name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr.18c1035d3da9278e namespace: kserve-ci-e2e-test resourceVersion: "15737" uid: eec8f38d-ee73-4026-a537-fa0a07d6120d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 69e78f05-d420-4352-bb49-9336e87fa5e8 kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-4aae5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:26Z" name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr.18c1035d4e22b9ab namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: e2b4c86a-68b2-4a25-8a33-cccddb07f92b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: 69e78f05-d420-4352-bb49-9336e87fa5e8 kind: Event lastTimestamp: "2026-07-10T19:10:27Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:10:27Z" name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr.18c1035d7af2694c namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 13356f02-2c60-4eb9-8030-956f32f54685 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 69e78f05-d420-4352-bb49-9336e87fa5e8 kind: Event lastTimestamp: "2026-07-10T19:10:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:27Z" name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr.18c1035d7c61fcd8 namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 70fd4df2-ceee-4c91-a094-ba56cadc31f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 69e78f05-d420-4352-bb49-9336e87fa5e8 kind: Event lastTimestamp: "2026-07-10T19:10:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:27Z" name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr.18c1035d7f39e859 namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: 0688a4c7-b838-47b9-bb32-b1a04eb7805a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 69e78f05-d420-4352-bb49-9336e87fa5e8 kind: Event lastTimestamp: "2026-07-10T19:10:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:27Z" name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr.18c1035d7fb0bf75 namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: 93a05aee-2c9f-453f-8386-d1b65b9c6a1f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 69e78f05-d420-4352-bb49-9336e87fa5e8 kind: Event lastTimestamp: "2026-07-10T19:10:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:27Z" name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr.18c1035d7fbbf364 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: a9b75271-8859-41fb-aafd-3be3cdf68965 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 69e78f05-d420-4352-bb49-9336e87fa5e8 kind: Event lastTimestamp: "2026-07-10T19:10:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:27Z" name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr.18c1035d8284f415 namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: 7d2c0513-d135-44a7-b823-aff2ad21a126 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 69e78f05-d420-4352-bb49-9336e87fa5e8 kind: Event lastTimestamp: "2026-07-10T19:10:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:27Z" name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr.18c1035d82f6b736 namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: c25506f5-8279-4d18-8956-c7f4fb8709f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:10:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 69e78f05-d420-4352-bb49-9336e87fa5e8 kind: Event lastTimestamp: "2026-07-10T19:11:05Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:10:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:05Z" name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr.18c1035dda961ac6 namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: 4bd6c152-c194-4ac2-bf6a-38f1aa79c75c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 69e78f05-d420-4352-bb49-9336e87fa5e8 kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:41Z" name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr.18c103de6955ab39 namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: abb53b2a-ef8e-4ad6-acaa-82ebe592f196 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 69e78f05-d420-4352-bb49-9336e87fa5e8 kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:41Z" name: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr.18c103de6956cf24 namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: 593e2f26-0b7c-4e80-b998-c07921d4e3bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4aae5-predictor-788b8dfbc4 namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 380adab9-0edf-49bf-816a-8a825c3a9078 kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: 'Created pod: error-404-isvc-4aae5-predictor-788b8dfbc4-7ffrr' metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:26Z" name: error-404-isvc-4aae5-predictor-788b8dfbc4.18c1035d3d3605eb namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: 7998dd5a-8138-4b22-a848-14e58ec56cad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4aae5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15721" uid: dbc7f731-9bad-4cfb-93e7-3b182c333f0c kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: Scaled up replica set error-404-isvc-4aae5-predictor-788b8dfbc4 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:26Z" name: error-404-isvc-4aae5-predictor.18c1035d3b918ca1 namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: d3add71b-7f70-4069-a8ca-71c6acb74d43 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:10:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4aae5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15724" uid: be092c92-dbee-4bbc-afe2-1a16a89b1577 kind: Event lastTimestamp: "2026-07-10T19:11:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:11Z" name: error-404-isvc-4aae5-predictor.18c10360ba1bbcff namespace: kserve-ci-e2e-test resourceVersion: "16383" uid: cb45f793-58b5-4a2c-8f10-98fdd843e224 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:10:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4aae5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15724" uid: be092c92-dbee-4bbc-afe2-1a16a89b1577 kind: Event lastTimestamp: "2026-07-10T19:11:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:11Z" name: error-404-isvc-4aae5-predictor.18c10360ba81cd87 namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: e7e198ce-8d96-4b5d-a78c-e6903a1356f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:10:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4aae5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: be092c92-dbee-4bbc-afe2-1a16a89b1577 kind: Event lastTimestamp: "2026-07-10T19:11:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:10:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:41Z" name: error-404-isvc-4aae5-predictor.18c10364392e2bf5 namespace: kserve-ci-e2e-test resourceVersion: "16683" uid: 68b5bace-34af-4117-871c-874b1353b034 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:10:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4aae5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: be092c92-dbee-4bbc-afe2-1a16a89b1577 kind: Event lastTimestamp: "2026-07-10T19:11:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:10:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:41Z" name: error-404-isvc-4aae5-predictor.18c103643997499b namespace: kserve-ci-e2e-test resourceVersion: "16684" uid: 4caca41f-cabb-44fc-b54e-d6b3823b2da5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: a49ae27e-404d-4378-9d42-e29a1bc36813 kind: Event lastTimestamp: "2026-07-10T19:11:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:15Z" name: error-404-isvc-4aae5.18c1035d3bbabe25 namespace: kserve-ci-e2e-test resourceVersion: "16423" uid: 620f7249-2cb7-4be2-b326-bfa12600afa8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: a49ae27e-404d-4378-9d42-e29a1bc36813 kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-4aae5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4aae5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:26Z" name: error-404-isvc-4aae5.18c1035d3c3c530c namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 0e29940a-ad50-4ef0-9770-9f954f42095a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: a49ae27e-404d-4378-9d42-e29a1bc36813 kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4aae5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:26Z" name: error-404-isvc-4aae5.18c1035d3c3c9281 namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: 80298cfa-7e00-4bfe-8918-8d4f0a504f39 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "16421" uid: a49ae27e-404d-4378-9d42-e29a1bc36813 kind: Event lastTimestamp: "2026-07-10T19:11:15Z" message: InferenceService [error-404-isvc-4aae5] is Ready metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:15Z" name: error-404-isvc-4aae5.18c103689644f485 namespace: kserve-ci-e2e-test resourceVersion: "16422" uid: 1f2974ec-a2db-44ea-b4fb-10a8e5eb324a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:30:16.555850Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f namespace: kserve-ci-e2e-test resourceVersion: "25746" uid: 9e8dabdf-ef51-425b-a17d-cfacffb419d3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:30:16Z" name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f.18c10472577a88c4 namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: d1b4c12f-2967-4378-99de-9d22760b8c73 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 9e8dabdf-ef51-425b-a17d-cfacffb419d3 kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-4b29b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:16Z" name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f.18c1047264ee72f5 namespace: kserve-ci-e2e-test resourceVersion: "25777" uid: 0e108ec4-ecad-4ee3-b9ee-c6e4703df414 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f namespace: kserve-ci-e2e-test resourceVersion: "25758" uid: 9e8dabdf-ef51-425b-a17d-cfacffb419d3 kind: Event lastTimestamp: "2026-07-10T19:30:17Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:30:17Z" name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f.18c1047294dab114 namespace: kserve-ci-e2e-test resourceVersion: "25794" uid: 96a7a748-e4ed-495d-8cbf-2c8a2a1aced7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 9e8dabdf-ef51-425b-a17d-cfacffb419d3 kind: Event lastTimestamp: "2026-07-10T19:30:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:17Z" name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f.18c104729652c4b7 namespace: kserve-ci-e2e-test resourceVersion: "25796" uid: 2e090f7d-4542-40a2-9a5f-80b3ef596fd7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 9e8dabdf-ef51-425b-a17d-cfacffb419d3 kind: Event lastTimestamp: "2026-07-10T19:30:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:17Z" name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f.18c104729a16371a namespace: kserve-ci-e2e-test resourceVersion: "25799" uid: d577f9f0-47d9-4fbd-b93c-83adc518cd38 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 9e8dabdf-ef51-425b-a17d-cfacffb419d3 kind: Event lastTimestamp: "2026-07-10T19:30:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:17Z" name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f.18c104729a934f39 namespace: kserve-ci-e2e-test resourceVersion: "25802" uid: 134d7fdf-5066-433d-aa56-d568341180ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 9e8dabdf-ef51-425b-a17d-cfacffb419d3 kind: Event lastTimestamp: "2026-07-10T19:30:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:17Z" name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f.18c104729a9d48ee namespace: kserve-ci-e2e-test resourceVersion: "25803" uid: a8080272-5b10-4e44-93e6-2bf2bceeb6d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 9e8dabdf-ef51-425b-a17d-cfacffb419d3 kind: Event lastTimestamp: "2026-07-10T19:30:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:17Z" name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f.18c104729d7e3270 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 230494d9-2aa0-47df-a34d-958ce8ac6064 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 9e8dabdf-ef51-425b-a17d-cfacffb419d3 kind: Event lastTimestamp: "2026-07-10T19:30:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:17Z" name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f.18c104729df1971b namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: 6eedab8b-b056-497a-addd-d82d2b4c3d15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-10T19:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 9e8dabdf-ef51-425b-a17d-cfacffb419d3 kind: Event lastTimestamp: "2026-07-10T19:30:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:55Z" name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f.18c10472d59f5856 namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: cced5759-3908-489f-a945-87f2d1003c8b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 9e8dabdf-ef51-425b-a17d-cfacffb419d3 kind: Event lastTimestamp: "2026-07-10T19:31:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:26Z" name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f.18c10482b2e840d1 namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: 53a777ae-5e22-4522-8085-90cf72a69bc8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 9e8dabdf-ef51-425b-a17d-cfacffb419d3 kind: Event lastTimestamp: "2026-07-10T19:31:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:26Z" name: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f.18c10482b2eb22e8 namespace: kserve-ci-e2e-test resourceVersion: "26641" uid: 36e9465d-6442-469b-9eb2-619ca3dfe969 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4b29b-predictor-7c6b8dbc64 namespace: kserve-ci-e2e-test resourceVersion: "25735" uid: db47f92d-1eab-4866-b13d-4bb067b7520e kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: 'Created pod: error-404-isvc-4b29b-predictor-7c6b8dbc64-6jf7f' metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:30:16Z" name: error-404-isvc-4b29b-predictor-7c6b8dbc64.18c1047256f2bfb3 namespace: kserve-ci-e2e-test resourceVersion: "25747" uid: f5e12512-93a4-4971-83b1-6d2cd163c6b5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4b29b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25729" uid: fd5e46f6-177f-43cf-82aa-d5957e323e1d kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: Scaled up replica set error-404-isvc-4b29b-predictor-7c6b8dbc64 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:30:16Z" name: error-404-isvc-4b29b-predictor.18c10472555c1db3 namespace: kserve-ci-e2e-test resourceVersion: "25738" uid: 37e65704-654a-448c-99ff-70f7587320a0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4b29b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25733" uid: 868d754b-2da3-460d-b6e8-5d2f9d8ecbee kind: Event lastTimestamp: "2026-07-10T19:30:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:30:46Z" name: error-404-isvc-4b29b-predictor.18c10475d37b5ab7 namespace: kserve-ci-e2e-test resourceVersion: "26196" uid: 0710b2b9-a286-48e1-9418-fc4ac184b66d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4b29b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25733" uid: 868d754b-2da3-460d-b6e8-5d2f9d8ecbee kind: Event lastTimestamp: "2026-07-10T19:30:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:30:46Z" name: error-404-isvc-4b29b-predictor.18c10475d3e47352 namespace: kserve-ci-e2e-test resourceVersion: "26197" uid: 24219443-aac6-4cdd-9312-bc8899a38cbe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:31:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4b29b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: 868d754b-2da3-460d-b6e8-5d2f9d8ecbee kind: Event lastTimestamp: "2026-07-10T19:31:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:16Z" name: error-404-isvc-4b29b-predictor.18c1047cd130f2a4 namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: cc155641-63b1-4c81-a494-10e7a2a49b99 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:31:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4b29b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: 868d754b-2da3-460d-b6e8-5d2f9d8ecbee kind: Event lastTimestamp: "2026-07-10T19:31:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:16Z" name: error-404-isvc-4b29b-predictor.18c1047cd1981f29 namespace: kserve-ci-e2e-test resourceVersion: "26479" uid: 5c1a6aab-9a39-4bc8-b952-1bbcdcbe5fe5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4b29b namespace: kserve-ci-e2e-test resourceVersion: "25718" uid: 638a7c30-276c-4a85-9a1c-78e9096d5c4c kind: Event lastTimestamp: "2026-07-10T19:31:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:05Z" name: error-404-isvc-4b29b.18c10472555f4e90 namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: af8a2157-d3fc-4c54-9314-17323449e424 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4b29b namespace: kserve-ci-e2e-test resourceVersion: "25718" uid: 638a7c30-276c-4a85-9a1c-78e9096d5c4c kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: 'Failed to update status for InferenceService "error-404-isvc-4b29b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4b29b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:30:16Z" name: error-404-isvc-4b29b.18c1047255dcbde1 namespace: kserve-ci-e2e-test resourceVersion: "25741" uid: e0925d08-c2a8-45c1-9fc4-b3ea64e711a4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4b29b namespace: kserve-ci-e2e-test resourceVersion: "25718" uid: 638a7c30-276c-4a85-9a1c-78e9096d5c4c kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4b29b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:30:16Z" name: error-404-isvc-4b29b.18c1047255dcfcd7 namespace: kserve-ci-e2e-test resourceVersion: "25744" uid: db1c79c1-4f66-479d-a65d-7b15b14ca3bb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4b29b namespace: kserve-ci-e2e-test resourceVersion: "26375" uid: 638a7c30-276c-4a85-9a1c-78e9096d5c4c kind: Event lastTimestamp: "2026-07-10T19:31:05Z" message: InferenceService [error-404-isvc-4b29b] is Ready metadata: creationTimestamp: "2026-07-10T19:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:05Z" name: error-404-isvc-4b29b.18c1047dccb35b73 namespace: kserve-ci-e2e-test resourceVersion: "26376" uid: 8dee7316-d58c-4bac-843f-135035a7dd0a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:21:27.785787Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz namespace: kserve-ci-e2e-test resourceVersion: "21532" uid: 03d4b1b6-53bd-44bd-822e-782b747038f4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:21:27Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz.18c103f73a53b3a9 namespace: kserve-ci-e2e-test resourceVersion: "21536" uid: 4355ea43-6f12-4ba4-8415-520491b1e38e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 03d4b1b6-53bd-44bd-822e-782b747038f4 kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-5e7fe-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:28Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz.18c103f747c8d2e3 namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: a3c81d04-9f3f-4150-bd3d-bfca641b62de reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz namespace: kserve-ci-e2e-test resourceVersion: "21542" uid: 03d4b1b6-53bd-44bd-822e-782b747038f4 kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:21:28Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz.18c103f778bc922c namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: 6f4268b8-beec-475f-bf52-35e3447a0346 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 03d4b1b6-53bd-44bd-822e-782b747038f4 kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:28Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz.18c103f77aa4d67f namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 9e2319f4-bc2a-4fea-9eef-510c987bcde7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 03d4b1b6-53bd-44bd-822e-782b747038f4 kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:28Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz.18c103f77caefa0a namespace: kserve-ci-e2e-test resourceVersion: "21576" uid: 9ab6ef19-9d84-4d42-ad9b-4977f987bd0c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 03d4b1b6-53bd-44bd-822e-782b747038f4 kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:28Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz.18c103f77d395744 namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: dfd1d413-9ce5-4a5a-9b6f-bc4330a34d5e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 03d4b1b6-53bd-44bd-822e-782b747038f4 kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:28Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz.18c103f77d45ecc3 namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: fe230c65-cb71-4443-bc4b-f20d0f4363f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 03d4b1b6-53bd-44bd-822e-782b747038f4 kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:28Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz.18c103f77fdcc209 namespace: kserve-ci-e2e-test resourceVersion: "21579" uid: 2a3707da-3003-4f26-a0e3-534f8941db92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 03d4b1b6-53bd-44bd-822e-782b747038f4 kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:28Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz.18c103f780a6c92c namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: c673889d-5783-478b-9af4-ba4a62a98294 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-10T19:21:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 03d4b1b6-53bd-44bd-822e-782b747038f4 kind: Event lastTimestamp: "2026-07-10T19:30:45Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:45Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz.18c103f7a8ff18a0 namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: 65fa3fdc-3737-4847-9f8a-75075d37e266 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 03d4b1b6-53bd-44bd-822e-782b747038f4 kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:42Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz.18c10478646c1b80 namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 130e49a9-9e45-4568-ad0e-d2b6e6420694 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 03d4b1b6-53bd-44bd-822e-782b747038f4 kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:42Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz.18c10478646ca5a0 namespace: kserve-ci-e2e-test resourceVersion: "26085" uid: 130ddbab-a5c3-4191-a28c-b08bfd99b410 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 03d4b1b6-53bd-44bd-822e-782b747038f4 kind: Event lastTimestamp: "2026-07-10T19:30:45Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:30:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:45Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz.18c104791d8711db namespace: kserve-ci-e2e-test resourceVersion: "26183" uid: e9492fbe-6051-4d2d-8ed0-a653cb1f1ccc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5e7fe-predictor-58dbcbf874 namespace: kserve-ci-e2e-test resourceVersion: "21525" uid: d005057e-6b30-4853-ab56-f78df0b1db08 kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: 'Created pod: error-404-isvc-5e7fe-predictor-58dbcbf874-cvkmz' metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:27Z" name: error-404-isvc-5e7fe-predictor-58dbcbf874.18c103f739bf83bf namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: 8314315e-3482-4c3e-8019-582a7d92d7cc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5e7fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21523" uid: 87cbc07c-8beb-4716-aad8-175408c597ff kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: Scaled up replica set error-404-isvc-5e7fe-predictor-58dbcbf874 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:27Z" name: error-404-isvc-5e7fe-predictor.18c103f737e70a94 namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: eb81b185-52fb-46b2-af2b-17fca3e23973 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5e7fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: dd3d1aab-1f0d-443f-8e25-2baf0858b048 kind: Event lastTimestamp: "2026-07-10T19:21:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:57Z" name: error-404-isvc-5e7fe-predictor.18c103fab66097bc namespace: kserve-ci-e2e-test resourceVersion: "21844" uid: 0a41dd6a-b7dc-465e-8fff-0fc2585c9426 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5e7fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: dd3d1aab-1f0d-443f-8e25-2baf0858b048 kind: Event lastTimestamp: "2026-07-10T19:21:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:57Z" name: error-404-isvc-5e7fe-predictor.18c103fab6ca491e namespace: kserve-ci-e2e-test resourceVersion: "21845" uid: 9e3097a4-bd89-4b45-8ad5-ea1824e43c3e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:22:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5e7fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21714" uid: dd3d1aab-1f0d-443f-8e25-2baf0858b048 kind: Event lastTimestamp: "2026-07-10T19:22:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:22:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:42Z" name: error-404-isvc-5e7fe-predictor.18c10401b40cdbfe namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 68ea5541-bd11-4d6a-bb74-76017ca061de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:22:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5e7fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21714" uid: dd3d1aab-1f0d-443f-8e25-2baf0858b048 kind: Event lastTimestamp: "2026-07-10T19:22:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:22:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:42Z" name: error-404-isvc-5e7fe-predictor.18c10401b47321e2 namespace: kserve-ci-e2e-test resourceVersion: "22277" uid: 8d484161-92ef-41ca-a6f9-c9d6aeb9f413 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5e7fe namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: d7d54e1a-e37b-46a2-b13e-cd60c284cbe4 kind: Event lastTimestamp: "2026-07-10T19:22:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:22:15Z" name: error-404-isvc-5e7fe.18c103f7381c9264 namespace: kserve-ci-e2e-test resourceVersion: "22028" uid: 15a9adbd-5b52-46c9-aa83-4375a714bde3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5e7fe namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: d7d54e1a-e37b-46a2-b13e-cd60c284cbe4 kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: 'Failed to update status for InferenceService "error-404-isvc-5e7fe": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5e7fe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:27Z" name: error-404-isvc-5e7fe.18c103f738a2719d namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: e3c859eb-9391-4600-b76b-93d7c74156e1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5e7fe namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: d7d54e1a-e37b-46a2-b13e-cd60c284cbe4 kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5e7fe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:27Z" name: error-404-isvc-5e7fe.18c103f738a2b825 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: 999c36f0-be3b-4ea8-838f-a2ee0ce004f4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5e7fe namespace: kserve-ci-e2e-test resourceVersion: "22026" uid: d7d54e1a-e37b-46a2-b13e-cd60c284cbe4 kind: Event lastTimestamp: "2026-07-10T19:22:15Z" message: InferenceService [error-404-isvc-5e7fe] is Ready metadata: creationTimestamp: "2026-07-10T19:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:22:15Z" name: error-404-isvc-5e7fe.18c10402640ddf3f namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: c18bc501-4f58-445b-a114-5d82c96e6bca reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:08:52.419241Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: 030651ea-b258-46f1-be66-6c973c0b82fe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:08:52Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm.18c103475af8df91 namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: 24e9e3f9-7464-4ef4-a106-957a224dcafb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm namespace: kserve-ci-e2e-test resourceVersion: "14683" uid: 030651ea-b258-46f1-be66-6c973c0b82fe kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:08:52Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm.18c10347757cd1c8 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: 0fe1208d-0e3c-4dcc-a74e-3358acc5347a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 030651ea-b258-46f1-be66-6c973c0b82fe kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1722" metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:08:52Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm.18c1034776d88fd9 namespace: kserve-ci-e2e-test resourceVersion: "14733" uid: 6f73c0b2-61f6-4719-b3db-d915a0e1fb47 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 030651ea-b258-46f1-be66-6c973c0b82fe kind: Event lastTimestamp: "2026-07-10T19:09:10Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1722" in 17.509s (17.509s including waiting). Image size: 1334625878 bytes.' metadata: creationTimestamp: "2026-07-10T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:10Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm.18c1034b8a7f4ab4 namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 52531102-988d-4237-889c-b0cfff379e7d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 030651ea-b258-46f1-be66-6c973c0b82fe kind: Event lastTimestamp: "2026-07-10T19:09:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:10Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm.18c1034b8e6d6626 namespace: kserve-ci-e2e-test resourceVersion: "14906" uid: 887e8b60-73bf-487f-a99e-81ce91042a91 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 030651ea-b258-46f1-be66-6c973c0b82fe kind: Event lastTimestamp: "2026-07-10T19:09:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:10Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm.18c1034b8effa88f namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: 3ecf8813-80f1-4bd8-9289-881643294b50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 030651ea-b258-46f1-be66-6c973c0b82fe kind: Event lastTimestamp: "2026-07-10T19:09:10Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-10T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:10Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm.18c1034b8f0c03f5 namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: b9a27443-00fb-4342-bf47-941c55fecdf3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 030651ea-b258-46f1-be66-6c973c0b82fe kind: Event lastTimestamp: "2026-07-10T19:09:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.61s (2.61s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-10T19:09:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:13Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm.18c1034c2aa48ee1 namespace: kserve-ci-e2e-test resourceVersion: "14932" uid: 709c10be-0537-4216-9b6b-cf421feb39f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 030651ea-b258-46f1-be66-6c973c0b82fe kind: Event lastTimestamp: "2026-07-10T19:09:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:09:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:13Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm.18c1034c2e59f615 namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: 5587ff3d-902b-4d64-ad50-3fddaa29baab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 030651ea-b258-46f1-be66-6c973c0b82fe kind: Event lastTimestamp: "2026-07-10T19:09:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:09:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:13Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm.18c1034c2ed2b5fc namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: fffaae47-0048-4d72-8600-15d2d039e4f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:09:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 030651ea-b258-46f1-be66-6c973c0b82fe kind: Event lastTimestamp: "2026-07-10T19:09:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:09:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:49Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm.18c1034c51e4ceea namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 4e5fe297-7bb8-4cae-a59a-365df09fcede reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 030651ea-b258-46f1-be66-6c973c0b82fe kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:26Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm.18c1035d36686f12 namespace: kserve-ci-e2e-test resourceVersion: "15700" uid: 3f140944-8b56-44fd-ae5f-3b75fbc8767f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 030651ea-b258-46f1-be66-6c973c0b82fe kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:26Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm.18c1035d3669b2fe namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: afdcb101-0f54-4d77-a3c4-afafc2996ae1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ac00e-predictor-6b6fb6bc7b namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: 0250a319-168d-4c29-b770-1525f3781924 kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: 'Created pod: error-404-isvc-ac00e-predictor-6b6fb6bc7b-dm4wm' metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:08:52Z" name: error-404-isvc-ac00e-predictor-6b6fb6bc7b.18c103475a8ea260 namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: bcaeb1bf-58f4-4a3a-8fe6-6b74a387fd64 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ac00e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 9e124a18-a23e-4203-bd8d-f186d16801ac kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: Scaled up replica set error-404-isvc-ac00e-predictor-6b6fb6bc7b from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:08:52Z" name: error-404-isvc-ac00e-predictor.18c1034757eceebe namespace: kserve-ci-e2e-test resourceVersion: "14666" uid: 7e07e37d-a124-492c-babb-3ffe18ffcc1d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:09:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ac00e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: d1e93968-7394-45d6-8ea5-55a59bcd937d kind: Event lastTimestamp: "2026-07-10T19:09:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:09:52Z" name: error-404-isvc-ac00e-predictor.18c1034ad63cc818 namespace: kserve-ci-e2e-test resourceVersion: "15308" uid: 44716b9a-72f1-4cc5-b646-1ccd05a8f6c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:09:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ac00e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: d1e93968-7394-45d6-8ea5-55a59bcd937d kind: Event lastTimestamp: "2026-07-10T19:09:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:09:52Z" name: error-404-isvc-ac00e-predictor.18c1034ad6a06e68 namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: 829d52df-a712-4169-bbae-787bee895c81 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:09:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ac00e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: d1e93968-7394-45d6-8ea5-55a59bcd937d kind: Event lastTimestamp: "2026-07-10T19:10:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:09:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:22Z" name: error-404-isvc-ac00e-predictor.18c10351d45a5fe3 namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: 96aa6168-f2b6-4f72-ac67-7b0e165a0b62 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:09:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ac00e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: d1e93968-7394-45d6-8ea5-55a59bcd937d kind: Event lastTimestamp: "2026-07-10T19:10:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:09:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:22Z" name: error-404-isvc-ac00e-predictor.18c10351d4c7bcbe namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: 48527912-cc07-49e2-b658-e5d5b935eca9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ac00e namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: 969d8f16-1453-4929-a787-b248c4896d07 kind: Event lastTimestamp: "2026-07-10T19:09:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:09:59Z" name: error-404-isvc-ac00e.18c1034757eb2067 namespace: kserve-ci-e2e-test resourceVersion: "15377" uid: 5f10779c-c990-4498-8b48-91ac0a30f03e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ac00e namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: 969d8f16-1453-4929-a787-b248c4896d07 kind: Event lastTimestamp: "2026-07-10T19:09:59Z" message: InferenceService [error-404-isvc-ac00e] is Ready metadata: creationTimestamp: "2026-07-10T19:09:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:09:59Z" name: error-404-isvc-ac00e.18c103570e5d9e79 namespace: kserve-ci-e2e-test resourceVersion: "15372" uid: f1918dab-659a-4656-92af-08234923563c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:20:51.670235Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: 1d8b4dd5-a9be-4ded-8f51-c3f498f541b0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d232d-predictor-6d6f664f96-bzwzh to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:20:51Z" name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh.18c103eed1ac24d9 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 7c48666f-8efa-41ad-b005-eda591fe5f34 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 1d8b4dd5-a9be-4ded-8f51-c3f498f541b0 kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d232d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:51Z" name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh.18c103eed9a1731c namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: a079cfab-93d4-4557-bafb-afcdd1a779bf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: 1d8b4dd5-a9be-4ded-8f51-c3f498f541b0 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:20:52Z" name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh.18c103ef0f5f5837 namespace: kserve-ci-e2e-test resourceVersion: "21075" uid: 8143b899-7dee-4a90-abea-3e5e3332ce8a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 1d8b4dd5-a9be-4ded-8f51-c3f498f541b0 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh.18c103ef11a76c0b namespace: kserve-ci-e2e-test resourceVersion: "21077" uid: 5a66a776-558a-41ca-85c5-a3db0eecc543 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 1d8b4dd5-a9be-4ded-8f51-c3f498f541b0 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh.18c103ef14d71564 namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: fc6c21cc-7937-40fe-bf07-e50b82f3a90e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 1d8b4dd5-a9be-4ded-8f51-c3f498f541b0 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh.18c103ef15f1d4f1 namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: 71bd1bae-50dd-4786-8fec-e93c3ab7fa52 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 1d8b4dd5-a9be-4ded-8f51-c3f498f541b0 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh.18c103ef15fc6599 namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: a71fd7be-9095-47cc-abe8-491dd09f851c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 1d8b4dd5-a9be-4ded-8f51-c3f498f541b0 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh.18c103ef18b77d6a namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: af100c53-82c8-4185-9252-3322449506e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 1d8b4dd5-a9be-4ded-8f51-c3f498f541b0 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh.18c103ef192cb59d namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: 64b29c98-dd5f-4db6-b05a-32af1e9ff44b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 1d8b4dd5-a9be-4ded-8f51-c3f498f541b0 kind: Event lastTimestamp: "2026-07-10T19:21:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:30Z" name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh.18c103ef7996e15d namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: 0dd422ee-3fac-4628-8ba2-7150a5517c12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 1d8b4dd5-a9be-4ded-8f51-c3f498f541b0 kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:16Z" name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh.18c104725aac58a5 namespace: kserve-ci-e2e-test resourceVersion: "25770" uid: 38c66bd0-cbe4-478e-abd5-49c14d419951 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 1d8b4dd5-a9be-4ded-8f51-c3f498f541b0 kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:16Z" name: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh.18c104725aace30d namespace: kserve-ci-e2e-test resourceVersion: "25771" uid: 6e9fcc64-0f65-4ec0-83fd-76d835d22d8f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d232d-predictor-6d6f664f96 namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: cdccb971-3d29-4d15-a919-dfcd30d01558 kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: 'Created pod: error-404-isvc-d232d-predictor-6d6f664f96-bzwzh' metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:51Z" name: error-404-isvc-d232d-predictor-6d6f664f96.18c103eed137a7fe namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: 66607881-ac41-4900-80aa-80cd6ede98c5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d232d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 844027fe-c213-4e58-8c5c-464194635e29 kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: Scaled up replica set error-404-isvc-d232d-predictor-6d6f664f96 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:51Z" name: error-404-isvc-d232d-predictor.18c103eecfa08988 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 00d49a48-dd6a-4f47-9e95-24120e7d9044 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d232d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21028" uid: 33476f9a-986b-484e-b384-d229ccae5cdb kind: Event lastTimestamp: "2026-07-10T19:21:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:21Z" name: error-404-isvc-d232d-predictor.18c103f24e1a0b86 namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: 8e45a5df-4f20-40f6-91ac-e5999e970f4a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d232d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21028" uid: 33476f9a-986b-484e-b384-d229ccae5cdb kind: Event lastTimestamp: "2026-07-10T19:21:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:21Z" name: error-404-isvc-d232d-predictor.18c103f24e7cf35b namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: db6ce7ff-4777-4027-9ee0-9c65341408b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d232d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21231" uid: 33476f9a-986b-484e-b384-d229ccae5cdb kind: Event lastTimestamp: "2026-07-10T19:22:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:06Z" name: error-404-isvc-d232d-predictor.18c103f94bd5742b namespace: kserve-ci-e2e-test resourceVersion: "21944" uid: 991c7dc2-0535-44b3-9fce-68e42a4e36c4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d232d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21231" uid: 33476f9a-986b-484e-b384-d229ccae5cdb kind: Event lastTimestamp: "2026-07-10T19:22:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:06Z" name: error-404-isvc-d232d-predictor.18c103f94c4f5754 namespace: kserve-ci-e2e-test resourceVersion: "21945" uid: 2afd5c6d-d6f4-4184-9901-72beb3ca4a53 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d232d namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 10350d27-ff6c-4082-ac45-e526284f12c1 kind: Event lastTimestamp: "2026-07-10T19:21:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:40Z" name: error-404-isvc-d232d.18c103eecfa513f2 namespace: kserve-ci-e2e-test resourceVersion: "21696" uid: f6207d5e-e574-447e-aca1-580e4f11bc6c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d232d namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 10350d27-ff6c-4082-ac45-e526284f12c1 kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-d232d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d232d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:51Z" name: error-404-isvc-d232d.18c103eecffd7065 namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: c033c335-5d89-472c-b191-2499a213d299 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d232d namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 10350d27-ff6c-4082-ac45-e526284f12c1 kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d232d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:51Z" name: error-404-isvc-d232d.18c103eecffdb076 namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: 3f6fb983-dd57-4b40-8e15-80a4472b8c08 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d232d namespace: kserve-ci-e2e-test resourceVersion: "21693" uid: 10350d27-ff6c-4082-ac45-e526284f12c1 kind: Event lastTimestamp: "2026-07-10T19:21:40Z" message: InferenceService [error-404-isvc-d232d] is Ready metadata: creationTimestamp: "2026-07-10T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:40Z" name: error-404-isvc-d232d.18c103fa34e6b5fe namespace: kserve-ci-e2e-test resourceVersion: "21694" uid: daa5a142-a3f0-44fd-955b-ee45fad0b7a1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:20:17.450354Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8 namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: fc4d78ba-478a-41a2-bb7c-77744b61bb2f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8 to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:20:17Z" name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8.18c103e6da021d23 namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 74273845-1d8e-4136-8064-5d04d1805ba6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8 namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: fc4d78ba-478a-41a2-bb7c-77744b61bb2f kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:20:17Z" name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8.18c103e6f38c48d0 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: 2950aed1-a30b-45ad-827c-80d9d80cc5ad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: fc4d78ba-478a-41a2-bb7c-77744b61bb2f kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:17Z" name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8.18c103e6f4f87950 namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: 960eeabb-a6a0-4fa3-a8c4-aa7968a07d6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: fc4d78ba-478a-41a2-bb7c-77744b61bb2f kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:17Z" name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8.18c103e6f80e82ae namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: 7684a8f7-d956-4f6b-8439-f83679de3471 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: fc4d78ba-478a-41a2-bb7c-77744b61bb2f kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:17Z" name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8.18c103e6f8814b38 namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: cad0fcb4-df50-441d-a9e4-3b92904c6148 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: fc4d78ba-478a-41a2-bb7c-77744b61bb2f kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:17Z" name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8.18c103e6f88acf8f namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: 5ccfdee9-efd4-4a68-b2d0-716c2c025e48 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: fc4d78ba-478a-41a2-bb7c-77744b61bb2f kind: Event lastTimestamp: "2026-07-10T19:20:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:20:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:18Z" name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8.18c103e6fc1d9ce6 namespace: kserve-ci-e2e-test resourceVersion: "20583" uid: e0bc22a9-532f-4442-9a0b-42709325667c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: fc4d78ba-478a-41a2-bb7c-77744b61bb2f kind: Event lastTimestamp: "2026-07-10T19:20:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:20:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:18Z" name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8.18c103e6fc9a1642 namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: 973d882c-066d-48e1-9c59-af6bd1305d40 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-10T19:20:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: fc4d78ba-478a-41a2-bb7c-77744b61bb2f kind: Event lastTimestamp: "2026-07-10T19:20:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:20:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:55Z" name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8.18c103e70d4eb052 namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 6d17b743-ab1c-4eb0-ab2c-39d686ba4695 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: fc4d78ba-478a-41a2-bb7c-77744b61bb2f kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:27Z" name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8.18c103f73055267b namespace: kserve-ci-e2e-test resourceVersion: "21471" uid: da19f373-b024-4dba-ba31-19ce149ddfc8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: fc4d78ba-478a-41a2-bb7c-77744b61bb2f kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:27Z" name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8.18c103f73055c024 namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: 12d399d3-3172-44c8-be67-11b018c959ac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: fc4d78ba-478a-41a2-bb7c-77744b61bb2f kind: Event lastTimestamp: "2026-07-10T19:21:30Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:30Z" name: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8.18c103f7d1c83ff7 namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: fb3c6e4f-b7fb-4166-bd93-2a95a17224ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d30ee-predictor-b984bbfb8 namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: 59128799-6972-4fdc-a374-996acd01b39f kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: 'Created pod: error-404-isvc-d30ee-predictor-b984bbfb8-jpqd8' metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:17Z" name: error-404-isvc-d30ee-predictor-b984bbfb8.18c103e6d95d39de namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: 7dfa0106-f162-445d-a0c3-0b15f68fab45 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d30ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: 2d6f0779-13a3-415d-af3e-80d58d2e3457 kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: Scaled up replica set error-404-isvc-d30ee-predictor-b984bbfb8 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:17Z" name: error-404-isvc-d30ee-predictor.18c103e6d7ad3434 namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: e8517da2-4f95-4f25-9b12-cb7c54f3f454 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:20:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d30ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 911ee8d8-4c11-4aa4-8874-9d4ad9906c6d kind: Event lastTimestamp: "2026-07-10T19:20:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:20:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:47Z" name: error-404-isvc-d30ee-predictor.18c103ea5622a0ef namespace: kserve-ci-e2e-test resourceVersion: "20907" uid: 8913a803-b004-4643-80f9-ddda76dd1708 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:20:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d30ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 911ee8d8-4c11-4aa4-8874-9d4ad9906c6d kind: Event lastTimestamp: "2026-07-10T19:20:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:20:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:47Z" name: error-404-isvc-d30ee-predictor.18c103ea567e4fb1 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: b5a2742a-d1f7-494d-9eef-43e582a25ae1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d30ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "20748" uid: 911ee8d8-4c11-4aa4-8874-9d4ad9906c6d kind: Event lastTimestamp: "2026-07-10T19:21:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:17Z" name: error-404-isvc-d30ee-predictor.18c103f153dfe23a namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: f08c3cba-4c71-4602-98cb-43e27152390f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d30ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "20748" uid: 911ee8d8-4c11-4aa4-8874-9d4ad9906c6d kind: Event lastTimestamp: "2026-07-10T19:21:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:17Z" name: error-404-isvc-d30ee-predictor.18c103f154478d3b namespace: kserve-ci-e2e-test resourceVersion: "21337" uid: 5b3d3fac-325a-486c-9981-d2e0081a542e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d30ee namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 0fee57fa-cecc-4b9d-b0ba-15e67c9897f8 kind: Event lastTimestamp: "2026-07-10T19:21:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:05Z" name: error-404-isvc-d30ee.18c103e6d7a3a03f namespace: kserve-ci-e2e-test resourceVersion: "21212" uid: 12658a7d-dc29-464b-bef0-beac975821a7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d30ee namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 0fee57fa-cecc-4b9d-b0ba-15e67c9897f8 kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: 'Failed to update status for InferenceService "error-404-isvc-d30ee": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d30ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:17Z" name: error-404-isvc-d30ee.18c103e6d81d883d namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 47ca3e2f-ece9-4b40-9622-4ad382a93345 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d30ee namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 0fee57fa-cecc-4b9d-b0ba-15e67c9897f8 kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d30ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:17Z" name: error-404-isvc-d30ee.18c103e6d81db561 namespace: kserve-ci-e2e-test resourceVersion: "20552" uid: 903ba546-2c4c-4f39-ac26-6fab47fddc7c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d30ee namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: 0fee57fa-cecc-4b9d-b0ba-15e67c9897f8 kind: Event lastTimestamp: "2026-07-10T19:21:05Z" message: InferenceService [error-404-isvc-d30ee] is Ready metadata: creationTimestamp: "2026-07-10T19:21:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:05Z" name: error-404-isvc-d30ee.18c103f2047fbced namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: d6380cf7-8fe4-4106-ba84-83bb4324afd4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:08:52.129727Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf to ip-10-0-134-251.ec2.internal metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:08:52Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c1034749b76544 namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: 844c7222-a099-4f5f-bb52-a3a6637e2ea1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:08:52Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c103475226cf82 namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: e29c2a80-f85f-493e-a983-a3f0a817418c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:53Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14609" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:08:53Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:08:53Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c10347867f838e namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: fa284a63-76c0-45b9-8075-1cbb58fe13d1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:08:53Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:08:53Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c10347880d88fb namespace: kserve-ci-e2e-test resourceVersion: "14745" uid: d336cf04-c3b0-4547-9efd-2dfcb37b29af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:08:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:08:53Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c103478a87ac2f namespace: kserve-ci-e2e-test resourceVersion: "14746" uid: 3337edfb-4401-4552-b82c-120ac5db9f90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:08:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:08:53Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c103478af5f15e namespace: kserve-ci-e2e-test resourceVersion: "14747" uid: a41f55ab-eafe-40e2-a172-8b1182b7d9e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:08:57Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" metadata: creationTimestamp: "2026-07-10T19:08:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:08:57Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c1034882d5c508 namespace: kserve-ci-e2e-test resourceVersion: "14793" uid: 33561668-384c-422e-8eb4-33fad8c43785 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:09:20Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" in 22.841s (22.841s including waiting). Image size: 1555838884 bytes.' metadata: creationTimestamp: "2026-07-10T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:20Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c1034dd4485f14 namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: 0fff9c2d-7e2c-47df-9b80-edad05ed3bf4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:09:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:20Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c1034dd9169c78 namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: ed3568fa-bcbc-4a1a-9c87-2e668699c770 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:09:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:20Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c1034dda165da9 namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: cf8b1041-a48c-492b-8f27-f88aa2949ff4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:09:20Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-10T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:20Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c1034dda22bde6 namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 538dd187-0bfd-4d00-9a6b-1c4979c5a58d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:09:22Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.274s (2.274s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-10T19:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:22Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c1034e61b53ca4 namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 9b16dbfa-89e3-45e7-9c33-cacd9952a684 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:09:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:22Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c1034e6495db2b namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: 3acc7ffb-a4dd-4475-a414-b9bec289bf9b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:09:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:22Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c1034e64f53d70 namespace: kserve-ci-e2e-test resourceVersion: "15050" uid: cfbb247b-1afc-4a1e-8aec-87ecf8b70784 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-10T19:09:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:10:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:20Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c1034ed62bdd84 namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: 4d496c36-fc8d-4eea-bb92-ae800800f9de reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c1036595f83db9 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: 6e3e9dc6-0315-467b-a9ab-5bdad1397b81 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c1036595f8d3a2 namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 4b7afacd-05f0-4e38-a201-9b7673b63ed5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 892aec3e-7a7a-4985-86fc-00d21e6d061a kind: Event lastTimestamp: "2026-07-10T19:11:05Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:05Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf.18c103665a741dc6 namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 253abde7-2e44-4322-a1e9-d17504c86e1a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-545bb6c58f namespace: kserve-ci-e2e-test resourceVersion: "14579" uid: fd45fc94-626a-4e52-a0b2-91b96281aede kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-545bb6c58f-smkcf' metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:08:52Z" name: isvc-sklearn-graph-1-predictor-545bb6c58f.18c1034749273280 namespace: kserve-ci-e2e-test resourceVersion: "14591" uid: c6ac0ff1-d3b8-4ccb-a6bd-a5445e22acaa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14578" uid: 3fe54900-a94b-4619-9e52-a9c1141512ef kind: Event lastTimestamp: "2026-07-10T19:08:51Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-545bb6c58f from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:08:51Z" name: isvc-sklearn-graph-1-predictor.18c1034740642781 namespace: kserve-ci-e2e-test resourceVersion: "14580" uid: 8cdce86c-e7dc-40b3-aa78-ab259af427de reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:09:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 6576c543-b10d-4abc-9c6b-7644fc2ec9ab kind: Event lastTimestamp: "2026-07-10T19:09:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:09:52Z" name: isvc-sklearn-graph-1-predictor.18c1034ac569a625 namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 8e0e580e-2575-4f0d-a703-f7c5e9b69863 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:09:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 6576c543-b10d-4abc-9c6b-7644fc2ec9ab kind: Event lastTimestamp: "2026-07-10T19:09:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:09:52Z" name: isvc-sklearn-graph-1-predictor.18c1034ac5cd7f51 namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: 5b979de7-a178-4a78-a48f-4078bcbe6492 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:10:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 6576c543-b10d-4abc-9c6b-7644fc2ec9ab kind: Event lastTimestamp: "2026-07-10T19:10:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:52Z" name: isvc-sklearn-graph-1-predictor.18c10358c1c1a8ce namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: a2f122d5-7f45-42f4-a0c2-1f1ba01a8c60 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:10:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 6576c543-b10d-4abc-9c6b-7644fc2ec9ab kind: Event lastTimestamp: "2026-07-10T19:10:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:52Z" name: isvc-sklearn-graph-1-predictor.18c10358c22565b9 namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: d2b01fb4-32f8-491a-b516-5d5b131d7c6a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: da6aac75-6c42-4ce1-9b64-14e4972ae60d kind: Event lastTimestamp: "2026-07-10T19:10:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:30Z" name: isvc-sklearn-graph-1.18c1034746494870 namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 217b4980-17c3-416d-8c77-971440bb7646 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: da6aac75-6c42-4ce1-9b64-14e4972ae60d kind: Event lastTimestamp: "2026-07-10T19:10:30Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-10T19:10:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:30Z" name: isvc-sklearn-graph-1.18c1035e39d80255 namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: 068ddef6-7361-4783-ab7d-0371ae786aac reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:08:52.526347Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14695" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:08:52Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c10347615b38e9 namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: e26800f0-9195-482d-b440-f96be46b0506 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:08:52Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c103477b496a5d namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: adf065b5-2217-49cf-8a7b-6003698c9e8a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:08:52Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c103477cc17314 namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: b1918dfa-e37c-4a32-ae18-2308b6a072b0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:09:10Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" in 17.354s (17.354s including waiting). Image size: 301881269 bytes.' metadata: creationTimestamp: "2026-07-10T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:10Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c1034b87259cea namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: b88b5497-bfe1-454a-a8e5-0742d3c5d944 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:09:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:10Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c1034b8bba1bd3 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 1756348d-1c49-4964-a57e-0d3166188e11 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:09:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:10Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c1034b8c6ca80a namespace: kserve-ci-e2e-test resourceVersion: "14903" uid: 9c2e6d96-a337-411c-a3b0-65c9191950ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:09:14Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" metadata: creationTimestamp: "2026-07-10T19:09:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:14Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c1034c8df24c29 namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: 6532ea7a-7e9e-4bf3-8c73-794955c94df2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:09:22Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" in 7.861s (7.861s including waiting). Image size: 1555838884 bytes.' metadata: creationTimestamp: "2026-07-10T19:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:22Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c1034e6283d732 namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 7a93454a-a83c-49ec-8a3f-d24942ce841c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:09:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:22Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c1034e695f2ed7 namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 7c0d04c3-6054-4e26-9f0a-aa91516b48c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:09:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:22Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c1034e6a8afed3 namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: ea949320-2fa1-45b7-b0d7-257620dfdcee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:09:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:22Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c1034e6aa21cf6 namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: b69c5a66-e26e-41fc-a7d6-5343a15e7a4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:09:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:22Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c1034e7054f5c7 namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: cd6d3614-9662-4e89-9c0e-1ae216a213a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:09:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:22Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c1034e7184833b namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: 5967a828-1756-4e3f-a3a1-47ef64844742 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T19:09:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:10:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:30Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c1034ee49fd1f9 namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: dff2d84a-eda8-4deb-be8a-778c21b7ae7d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c10365926aa065 namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: b5b11516-e1d5-468c-a156-1e73acd63264 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c10365926bd337 namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 6c99a3e4-f229-44a8-817b-0fc744c6eae9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 84fdc5e7-4a67-4ee2-8b3f-3fa819d62c95 kind: Event lastTimestamp: "2026-07-10T19:11:05Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:05Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z.18c1036668da3b1b namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 58dd2705-9ca2-4068-b2a0-d776943d043e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-cb544c5dc namespace: kserve-ci-e2e-test resourceVersion: "14689" uid: 6b99ae8b-9da7-44ae-ba24-7b2d7e3f4fea kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-cb544c5dc-5l29z' metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:08:52Z" name: isvc-sklearn-graph-2-predictor-cb544c5dc.18c1034760f40f0b namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: b5f98f25-941b-491b-b513-a5e1776337a3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14687" uid: a076a90d-a6e2-424c-8d89-ae602ae6d278 kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-cb544c5dc from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:08:52Z" name: isvc-sklearn-graph-2-predictor.18c103475e0595be namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: 0a6b1bda-3ed3-4b0e-a143-73d68d782870 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:09:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: c330db53-cfa5-428d-bc8e-77337d49ff6e kind: Event lastTimestamp: "2026-07-10T19:09:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:09:52Z" name: isvc-sklearn-graph-2-predictor.18c1034adc5dde42 namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 599b3ba6-abe6-4b9f-9445-fa68b6c793a0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:09:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: c330db53-cfa5-428d-bc8e-77337d49ff6e kind: Event lastTimestamp: "2026-07-10T19:09:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:09:52Z" name: isvc-sklearn-graph-2-predictor.18c1034adcc19d7d namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: 3e5d27b4-1742-4c58-95ef-1ada1b6cf885 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:10:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: c330db53-cfa5-428d-bc8e-77337d49ff6e kind: Event lastTimestamp: "2026-07-10T19:10:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:52Z" name: isvc-sklearn-graph-2-predictor.18c10358d76f109f namespace: kserve-ci-e2e-test resourceVersion: "16045" uid: 5b84b2de-9764-48f8-a9db-deef3633ea25 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:10:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: c330db53-cfa5-428d-bc8e-77337d49ff6e kind: Event lastTimestamp: "2026-07-10T19:10:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:52Z" name: isvc-sklearn-graph-2-predictor.18c10358d7d4a69a namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 4d02089e-eb97-42e1-8937-cc8b73edb02c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: 3a08ff81-0eaa-461a-82e5-56c194ecc2bd kind: Event lastTimestamp: "2026-07-10T19:10:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:40Z" name: isvc-sklearn-graph-2.18c103475e088725 namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 989636a5-c877-4735-a44f-20dbc6ec9486 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 3a08ff81-0eaa-461a-82e5-56c194ecc2bd kind: Event lastTimestamp: "2026-07-10T19:10:40Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-10T19:10:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:40Z" name: isvc-sklearn-graph-2.18c103609bebd509 namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: a148790e-2a19-440e-a195-121a3953a8d2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:08:52.329573Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14648" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-hkw68 to ip-10-0-134-251.ec2.internal metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:08:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c1034755a09f15 namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 9109bbb9-d55e-49d0-96d7-574c6dde1427 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14663" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:08:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c103476eb7020c namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: a97cd3eb-acfc-4d58-962e-c22eb4078b84 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:08:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c10347701793d9 namespace: kserve-ci-e2e-test resourceVersion: "14722" uid: 64c77513-dbf1-4023-adf7-e194874d6e0b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:08:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c10347732f5dcd namespace: kserve-ci-e2e-test resourceVersion: "14726" uid: c5b86e55-76fc-4824-810f-5d7785642df2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:08:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c10347739cefff namespace: kserve-ci-e2e-test resourceVersion: "14727" uid: 4781e525-7ba3-4920-898f-51e985816c90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:08:57Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-10T19:08:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:08:57Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c1034882f1d487 namespace: kserve-ci-e2e-test resourceVersion: "14794" uid: ea5f2158-10c0-47a4-89cb-46187a558a50 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:09:20Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 23.424s (23.424s including waiting). Image size: 1305567991 bytes.' metadata: creationTimestamp: "2026-07-10T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c1034df720faf4 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: 5b1af2a0-30b6-45cc-adb8-2fe787663cb8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:09:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c1034dfe9d5e0d namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: f7a1c90f-32b4-49b6-86d6-577f56589213 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:09:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c1034dffe40820 namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: 7a026048-5bbb-4678-9513-a01ac5c6aa91 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:09:20Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-10T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c1034dfff3d329 namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: 26ee2f03-cce3-492c-9517-cf729ef0bcbd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:09:22Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 1.629s (1.629s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-10T19:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c1034e610dcb9b namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 126abf8c-4f5d-4c34-bfa7-1f047fe248ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:09:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c1034e647f98d8 namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: 64dea934-7ddd-4ef6-bf2a-25427e751e16 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:09:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c1034e64ebb1e6 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 9413e87e-7822-4be6-9f3d-7b1596186d03 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-10T19:09:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:10:10Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c1034ed62bfabf namespace: kserve-ci-e2e-test resourceVersion: "15471" uid: 3841130a-6453-461e-ab2e-c20051503040 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c103658ff8c4fb namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: fd35f38a-c3e3-4cc9-990f-53d7dd020aae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hkw68 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 5a3bb143-a851-45dd-a549-894efa39b4ce kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-hkw68.18c103658ffd6a7b namespace: kserve-ci-e2e-test resourceVersion: "16169" uid: 394f692b-7236-4e2d-98f9-795a4087def7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-251.ec2.internal source: component: kubelet host: ip-10-0-134-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14635" uid: 380e0a27-8627-4564-9c50-13f2a2e4a475 kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-hkw68' metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:08:52Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c103475535d70f namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 89caf3c2-e389-4ec9-a2a7-df9f4a12ea6d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14633" uid: f3b51b46-ae50-41f2-ae59-0a7b7a240512 kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:08:52Z" name: isvc-xgboost-graph-predictor.18c103475212c7b6 namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: 4e35567e-5d9c-4b24-808f-e64a80abf5c1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:09:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14634" uid: 0295bc89-cb0e-4833-966c-b1fc4b665091 kind: Event lastTimestamp: "2026-07-10T19:09:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:09:52Z" name: isvc-xgboost-graph-predictor.18c1034ad099ad99 namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: 661e02cb-9b90-4645-8c72-4e4580c8249f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:09:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14634" uid: 0295bc89-cb0e-4833-966c-b1fc4b665091 kind: Event lastTimestamp: "2026-07-10T19:09:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:09:52Z" name: isvc-xgboost-graph-predictor.18c1034ad1028bce namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: 4b6926ae-0238-44e5-a53c-b9717f099ac0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:10:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: 0295bc89-cb0e-4833-966c-b1fc4b665091 kind: Event lastTimestamp: "2026-07-10T19:10:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:52Z" name: isvc-xgboost-graph-predictor.18c10358ccc2302b namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: f9c5f1e0-e554-4b31-bd4c-30a4ff98359c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:10:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: 0295bc89-cb0e-4833-966c-b1fc4b665091 kind: Event lastTimestamp: "2026-07-10T19:10:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:52Z" name: isvc-xgboost-graph-predictor.18c10358cd2a39f0 namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: 06d0c693-67a0-49a6-acf0-8b966b740b43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 47af23da-b25e-41f4-91c3-4f1f0fde5d4d kind: Event lastTimestamp: "2026-07-10T19:10:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:20Z" name: isvc-xgboost-graph.18c103475217d346 namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: b6fc5468-aa47-4014-8c47-7d546cebd79c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: 47af23da-b25e-41f4-91c3-4f1f0fde5d4d kind: Event lastTimestamp: "2026-07-10T19:10:20Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-10T19:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:20Z" name: isvc-xgboost-graph.18c1035be626c9ee namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: af5b55ee-2aa7-4cea-a1a9-e5e6ba88f568 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:10:51.967637Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7f4bd5d665-zszjv namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: bc43bd7e-4b98-436b-bbd2-55d6f2c7b384 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-7f4bd5d665-zszjv to ip-10-0-131-123.ec2.internal metadata: creationTimestamp: "2026-07-10T19:10:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:10:51Z" name: model-chainer-7f4bd5d665-zszjv.18c10363309c9732 namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: 16418f56-09dc-4d64-b47f-0c9a77d554dd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:52Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7f4bd5d665-zszjv namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: bc43bd7e-4b98-436b-bbd2-55d6f2c7b384 kind: Event lastTimestamp: "2026-07-10T19:10:52Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:10:52Z" name: model-chainer-7f4bd5d665-zszjv.18c103634a00d0dc namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: e9b93380-71d0-4f25-be52-e6002e9d29a8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7f4bd5d665-zszjv namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: bc43bd7e-4b98-436b-bbd2-55d6f2c7b384 kind: Event lastTimestamp: "2026-07-10T19:10:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" already present on machine metadata: creationTimestamp: "2026-07-10T19:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:52Z" name: model-chainer-7f4bd5d665-zszjv.18c103634bf195a8 namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: fee9872d-da16-4122-974b-4086663dc4fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7f4bd5d665-zszjv namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: bc43bd7e-4b98-436b-bbd2-55d6f2c7b384 kind: Event lastTimestamp: "2026-07-10T19:10:52Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-10T19:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:52Z" name: model-chainer-7f4bd5d665-zszjv.18c103634f0696cb namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: 1a134554-d619-47c6-9c3d-4231d6136c92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7f4bd5d665-zszjv namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: bc43bd7e-4b98-436b-bbd2-55d6f2c7b384 kind: Event lastTimestamp: "2026-07-10T19:10:52Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-10T19:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:52Z" name: model-chainer-7f4bd5d665-zszjv.18c103634f7b8713 namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 28a7a33e-e0e9-4033-bdd2-7ce256451694 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7f4bd5d665-zszjv namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: bc43bd7e-4b98-436b-bbd2-55d6f2c7b384 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: model-chainer-7f4bd5d665-zszjv.18c103658b66a6e5 namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: f162e87f-6d2a-48dd-9ac8-1a1bff488ed1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:11:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7f4bd5d665-zszjv namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: bc43bd7e-4b98-436b-bbd2-55d6f2c7b384 kind: Event lastTimestamp: "2026-07-10T19:11:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:29Z" name: model-chainer-7f4bd5d665-zszjv.18c103660c6b3fe5 namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: 29bc434c-0ac9-42ac-aa45-7e204ab6f403 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-7f4bd5d665 namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: b5afd3e3-70d3-442a-a0ee-cf94a71fd0b0 kind: Event lastTimestamp: "2026-07-10T19:10:51Z" message: 'Created pod: model-chainer-7f4bd5d665-zszjv' metadata: creationTimestamp: "2026-07-10T19:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:51Z" name: model-chainer-7f4bd5d665.18c10363301819d1 namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: cb1246c5-9fd4-4523-84c7-d1bfb0973b45 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: 67fe6732-2a05-4302-abd9-cb616e182cd5 kind: Event lastTimestamp: "2026-07-10T19:10:51Z" message: Scaled up replica set model-chainer-7f4bd5d665 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:51Z" name: model-chainer.18c103632ef96026 namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: dbc02b9d-c5ed-476c-92e2-69692cb81b14 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 8a26f877-b291-41ca-9c8b-787069217213 kind: Event lastTimestamp: "2026-07-10T19:10:52Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-10T19:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:52Z" name: model-chainer.18c103633543a527 namespace: kserve-ci-e2e-test resourceVersion: "16028" uid: 280b57b5-88ba-4a4a-8981-4671b2a508f1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:12:02.311710Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-31f26-7d88b899d6-c2gdl namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: fc35791b-1a59-4a4e-a6aa-d3020924b710 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-31f26-7d88b899d6-c2gdl to ip-10-0-131-123.ec2.internal metadata: creationTimestamp: "2026-07-10T19:12:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:12:02Z" name: sequence-graph-31f26-7d88b899d6-c2gdl.18c1037391721222 namespace: kserve-ci-e2e-test resourceVersion: "16869" uid: a87f069f-7d30-4446-b893-3453f7998317 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-31f26-7d88b899d6-c2gdl namespace: kserve-ci-e2e-test resourceVersion: "16865" uid: fc35791b-1a59-4a4e-a6aa-d3020924b710 kind: Event lastTimestamp: "2026-07-10T19:12:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-31f26-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:12:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:12:02Z" name: sequence-graph-31f26-7d88b899d6-c2gdl.18c10373a0c33b26 namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: 20a8c602-2202-46b0-acb6-7b840e386655 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:03Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-31f26-7d88b899d6-c2gdl namespace: kserve-ci-e2e-test resourceVersion: "16873" uid: fc35791b-1a59-4a4e-a6aa-d3020924b710 kind: Event lastTimestamp: "2026-07-10T19:12:03Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:12:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:12:03Z" name: sequence-graph-31f26-7d88b899d6-c2gdl.18c10373ce45d7e2 namespace: kserve-ci-e2e-test resourceVersion: "16894" uid: 20982fbe-fdb1-4afa-903e-7c7ae496affb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-31f26} kind: Pod name: sequence-graph-31f26-7d88b899d6-c2gdl namespace: kserve-ci-e2e-test resourceVersion: "16865" uid: fc35791b-1a59-4a4e-a6aa-d3020924b710 kind: Event lastTimestamp: "2026-07-10T19:12:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" already present on machine metadata: creationTimestamp: "2026-07-10T19:12:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:12:03Z" name: sequence-graph-31f26-7d88b899d6-c2gdl.18c10373cfbf66ac namespace: kserve-ci-e2e-test resourceVersion: "16896" uid: 397e94bd-9524-49b7-aae6-d1579d8f05c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-31f26} kind: Pod name: sequence-graph-31f26-7d88b899d6-c2gdl namespace: kserve-ci-e2e-test resourceVersion: "16865" uid: fc35791b-1a59-4a4e-a6aa-d3020924b710 kind: Event lastTimestamp: "2026-07-10T19:12:03Z" message: 'Created container: sequence-graph-31f26' metadata: creationTimestamp: "2026-07-10T19:12:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:12:03Z" name: sequence-graph-31f26-7d88b899d6-c2gdl.18c10373d2a9ce25 namespace: kserve-ci-e2e-test resourceVersion: "16897" uid: 44bcf406-c35f-4c19-8c9c-a8fe283d369b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-31f26} kind: Pod name: sequence-graph-31f26-7d88b899d6-c2gdl namespace: kserve-ci-e2e-test resourceVersion: "16865" uid: fc35791b-1a59-4a4e-a6aa-d3020924b710 kind: Event lastTimestamp: "2026-07-10T19:12:03Z" message: Started container sequence-graph-31f26 metadata: creationTimestamp: "2026-07-10T19:12:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:12:03Z" name: sequence-graph-31f26-7d88b899d6-c2gdl.18c10373d31330b3 namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: 570178eb-8569-4e6a-b16f-0da70a9e6fc1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-31f26} kind: Pod name: sequence-graph-31f26-7d88b899d6-c2gdl namespace: kserve-ci-e2e-test resourceVersion: "16865" uid: fc35791b-1a59-4a4e-a6aa-d3020924b710 kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: Stopping container sequence-graph-31f26 metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:17Z" name: sequence-graph-31f26-7d88b899d6-c2gdl.18c103e6c7ac2141 namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: 746ef92e-f24c-4cfe-989b-4d6ee31960fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:20:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-31f26} kind: Pod name: sequence-graph-31f26-7d88b899d6-c2gdl namespace: kserve-ci-e2e-test resourceVersion: "16865" uid: fc35791b-1a59-4a4e-a6aa-d3020924b710 kind: Event lastTimestamp: "2026-07-10T19:20:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:20:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:44Z" name: sequence-graph-31f26-7d88b899d6-c2gdl.18c103e757162a65 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 3296ad4f-c9f1-4525-b2ae-87018a5b4ac4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-31f26-7d88b899d6 namespace: kserve-ci-e2e-test resourceVersion: "16860" uid: de9ffc7b-95f5-47ed-a191-a3718c6b414b kind: Event lastTimestamp: "2026-07-10T19:12:02Z" message: 'Created pod: sequence-graph-31f26-7d88b899d6-c2gdl' metadata: creationTimestamp: "2026-07-10T19:12:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:02Z" name: sequence-graph-31f26-7d88b899d6.18c1037390fd6115 namespace: kserve-ci-e2e-test resourceVersion: "16866" uid: 71c3fad4-10dc-4c9c-bb7b-e72fffd926c7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16858" uid: 230ecb90-0441-490f-b623-a3784e45fce2 kind: Event lastTimestamp: "2026-07-10T19:12:02Z" message: Scaled up replica set sequence-graph-31f26-7d88b899d6 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:12:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:02Z" name: sequence-graph-31f26.18c1037390177e0c namespace: kserve-ci-e2e-test resourceVersion: "16861" uid: b4175d8d-8caf-40ca-ada2-472efe8b1d2a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16876" uid: df89114d-1aca-46f1-a721-0cc662f11cb8 kind: Event lastTimestamp: "2026-07-10T19:12:02Z" message: InferenceGraph [sequence-graph-31f26] is Ready metadata: creationTimestamp: "2026-07-10T19:12:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:12:02Z" name: sequence-graph-31f26.18c1037396246673 namespace: kserve-ci-e2e-test resourceVersion: "16878" uid: 3b252b4f-6e38-4171-baa3-d53508859286 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16853" uid: df89114d-1aca-46f1-a721-0cc662f11cb8 kind: Event lastTimestamp: "2026-07-10T19:12:02Z" message: 'Failed to update status for InferenceGraph "sequence-graph-31f26": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-31f26": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:12:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:12:02Z" name: sequence-graph-31f26.18c1037398b461c5 namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 7dfd8b23-28bf-4455-b21e-1a2d88c0d6af reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16853" uid: df89114d-1aca-46f1-a721-0cc662f11cb8 kind: Event lastTimestamp: "2026-07-10T19:12:02Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-31f26": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:12:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:12:02Z" name: sequence-graph-31f26.18c1037398b4b59e namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: e40bdfd0-aa7e-400d-ad4a-ae5e5c899507 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16859" uid: d28d80b6-d1ec-446e-9f5d-fc6a6822996b kind: Event lastTimestamp: "2026-07-10T19:12:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:12:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:17Z" name: sequence-graph-31f26.18c103770e98c4a6 namespace: kserve-ci-e2e-test resourceVersion: "17005" uid: e4fd1609-56f9-4025-a133-787ca1760089 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16859" uid: d28d80b6-d1ec-446e-9f5d-fc6a6822996b kind: Event lastTimestamp: "2026-07-10T19:12:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:12:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:17Z" name: sequence-graph-31f26.18c103770f03134e namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: d19204c3-19a1-4f69-8c1b-354684d32665 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-31f26 namespace: kserve-ci-e2e-test resourceVersion: "17006" uid: d28d80b6-d1ec-446e-9f5d-fc6a6822996b kind: Event lastTimestamp: "2026-07-10T19:12:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:12:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:32Z" name: sequence-graph-31f26.18c1037a8e6eee36 namespace: kserve-ci-e2e-test resourceVersion: "17117" uid: 2135625f-0d8d-4c11-9aa8-3acb0d092c22 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-31f26 namespace: kserve-ci-e2e-test resourceVersion: "17006" uid: d28d80b6-d1ec-446e-9f5d-fc6a6822996b kind: Event lastTimestamp: "2026-07-10T19:12:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:12:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:32Z" name: sequence-graph-31f26.18c1037a8f5aa80e namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 5acf42e4-d272-4d09-b9df-b4ce74eefb55 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:22:27.754977Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-5e7fe-5586b95c8c-pmgrj namespace: kserve-ci-e2e-test resourceVersion: "22137" uid: 6c937493-5f66-41aa-ab8d-53963ca388c8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-5e7fe-5586b95c8c-pmgrj to ip-10-0-131-123.ec2.internal metadata: creationTimestamp: "2026-07-10T19:22:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:22:27Z" name: sequence-graph-5e7fe-5586b95c8c-pmgrj.18c1040530c500b6 namespace: kserve-ci-e2e-test resourceVersion: "22142" uid: 5cbd6640-2b9e-4b17-9772-187215e83cd3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:27Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-5e7fe-5586b95c8c-pmgrj namespace: kserve-ci-e2e-test resourceVersion: "22138" uid: 6c937493-5f66-41aa-ab8d-53963ca388c8 kind: Event lastTimestamp: "2026-07-10T19:22:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-5e7fe-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:27Z" name: sequence-graph-5e7fe-5586b95c8c-pmgrj.18c1040538b2dbbd namespace: kserve-ci-e2e-test resourceVersion: "22154" uid: b3da203e-e906-40be-86f6-dc1c949a9a1f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:28Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-5e7fe-5586b95c8c-pmgrj namespace: kserve-ci-e2e-test resourceVersion: "22146" uid: 6c937493-5f66-41aa-ab8d-53963ca388c8 kind: Event lastTimestamp: "2026-07-10T19:22:28Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:22:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:22:28Z" name: sequence-graph-5e7fe-5586b95c8c-pmgrj.18c104056dda58bd namespace: kserve-ci-e2e-test resourceVersion: "22159" uid: f250bdd1-cc7f-41ca-94bd-e9281873ba65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5e7fe} kind: Pod name: sequence-graph-5e7fe-5586b95c8c-pmgrj namespace: kserve-ci-e2e-test resourceVersion: "22138" uid: 6c937493-5f66-41aa-ab8d-53963ca388c8 kind: Event lastTimestamp: "2026-07-10T19:22:28Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" already present on machine metadata: creationTimestamp: "2026-07-10T19:22:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:28Z" name: sequence-graph-5e7fe-5586b95c8c-pmgrj.18c104056f29c68d namespace: kserve-ci-e2e-test resourceVersion: "22163" uid: d0da5f18-3b53-4297-a786-f1dbad20e813 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5e7fe} kind: Pod name: sequence-graph-5e7fe-5586b95c8c-pmgrj namespace: kserve-ci-e2e-test resourceVersion: "22138" uid: 6c937493-5f66-41aa-ab8d-53963ca388c8 kind: Event lastTimestamp: "2026-07-10T19:22:28Z" message: 'Created container: sequence-graph-5e7fe' metadata: creationTimestamp: "2026-07-10T19:22:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:28Z" name: sequence-graph-5e7fe-5586b95c8c-pmgrj.18c1040571e4f785 namespace: kserve-ci-e2e-test resourceVersion: "22165" uid: 755192a7-ee92-47fe-a2b4-9e4451eb8567 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5e7fe} kind: Pod name: sequence-graph-5e7fe-5586b95c8c-pmgrj namespace: kserve-ci-e2e-test resourceVersion: "22138" uid: 6c937493-5f66-41aa-ab8d-53963ca388c8 kind: Event lastTimestamp: "2026-07-10T19:22:28Z" message: Started container sequence-graph-5e7fe metadata: creationTimestamp: "2026-07-10T19:22:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:28Z" name: sequence-graph-5e7fe-5586b95c8c-pmgrj.18c10405724f4c47 namespace: kserve-ci-e2e-test resourceVersion: "22166" uid: bbefe17a-52be-4eb3-9222-40ac92a708d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5e7fe} kind: Pod name: sequence-graph-5e7fe-5586b95c8c-pmgrj namespace: kserve-ci-e2e-test resourceVersion: "22138" uid: 6c937493-5f66-41aa-ab8d-53963ca388c8 kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: Stopping container sequence-graph-5e7fe metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:42Z" name: sequence-graph-5e7fe-5586b95c8c-pmgrj.18c104785ab4dedf namespace: kserve-ci-e2e-test resourceVersion: "26032" uid: 18a3fec2-7f9b-4e5d-87d0-e68273c9bb2f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:30:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5e7fe} kind: Pod name: sequence-graph-5e7fe-5586b95c8c-pmgrj namespace: kserve-ci-e2e-test resourceVersion: "22138" uid: 6c937493-5f66-41aa-ab8d-53963ca388c8 kind: Event lastTimestamp: "2026-07-10T19:31:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:30:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:10Z" name: sequence-graph-5e7fe-5586b95c8c-pmgrj.18c10478ff2a60ee namespace: kserve-ci-e2e-test resourceVersion: "26406" uid: 53cd2955-b076-4e07-a403-c542115bf678 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-5e7fe-5586b95c8c namespace: kserve-ci-e2e-test resourceVersion: "22134" uid: 0868f04f-eecd-43f9-a451-f9590c42c577 kind: Event lastTimestamp: "2026-07-10T19:22:27Z" message: 'Created pod: sequence-graph-5e7fe-5586b95c8c-pmgrj' metadata: creationTimestamp: "2026-07-10T19:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:27Z" name: sequence-graph-5e7fe-5586b95c8c.18c10405305da610 namespace: kserve-ci-e2e-test resourceVersion: "22139" uid: 10044bd9-6567-4388-8531-20b1d8c18a95 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-5e7fe namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: 4a581ebe-6174-472f-9a26-4f093b2a25d6 kind: Event lastTimestamp: "2026-07-10T19:22:27Z" message: Scaled up replica set sequence-graph-5e7fe-5586b95c8c from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:27Z" name: sequence-graph-5e7fe.18c104052f955130 namespace: kserve-ci-e2e-test resourceVersion: "22135" uid: 67e90ee5-1346-486a-b8f2-9081137e16ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-5e7fe namespace: kserve-ci-e2e-test resourceVersion: "22150" uid: 41cb9134-81f7-44d7-8170-a009ed304c79 kind: Event lastTimestamp: "2026-07-10T19:22:27Z" message: InferenceGraph [sequence-graph-5e7fe] is Ready metadata: creationTimestamp: "2026-07-10T19:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:22:27Z" name: sequence-graph-5e7fe.18c10405350d469c namespace: kserve-ci-e2e-test resourceVersion: "22152" uid: 5abdb864-d695-415a-b94c-fb6ffc68a398 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:22:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-5e7fe namespace: kserve-ci-e2e-test resourceVersion: "22133" uid: f59e9acb-ec3b-446e-911a-793050f68872 kind: Event lastTimestamp: "2026-07-10T19:22:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:22:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:57Z" name: sequence-graph-5e7fe.18c10408ae27a956 namespace: kserve-ci-e2e-test resourceVersion: "22382" uid: 8c6aa2e9-4810-45db-8d96-91edfaaeb25f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:22:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-5e7fe namespace: kserve-ci-e2e-test resourceVersion: "22133" uid: f59e9acb-ec3b-446e-911a-793050f68872 kind: Event lastTimestamp: "2026-07-10T19:22:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:22:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:57Z" name: sequence-graph-5e7fe.18c10408ae876b90 namespace: kserve-ci-e2e-test resourceVersion: "22383" uid: 13de71a7-38d9-47c7-9d79-0652e1e7e050 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:21:17.360577Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d30ee-6b65dbc99c-sjpgv namespace: kserve-ci-e2e-test resourceVersion: "21317" uid: 13ee2214-2cf8-46c3-985f-7335f1bc1aaf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-d30ee-6b65dbc99c-sjpgv to ip-10-0-131-123.ec2.internal metadata: creationTimestamp: "2026-07-10T19:21:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:21:17Z" name: sequence-graph-d30ee-6b65dbc99c-sjpgv.18c103f4ccef9eeb namespace: kserve-ci-e2e-test resourceVersion: "21323" uid: dbaeeb73-6a12-4a62-a580-cc2cf45eb80f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:17Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d30ee-6b65dbc99c-sjpgv namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: 13ee2214-2cf8-46c3-985f-7335f1bc1aaf kind: Event lastTimestamp: "2026-07-10T19:21:17Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:21:17Z" name: sequence-graph-d30ee-6b65dbc99c-sjpgv.18c103f4e650f037 namespace: kserve-ci-e2e-test resourceVersion: "21343" uid: f1985996-2df1-4b63-bda5-b9ac5716e723 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d30ee} kind: Pod name: sequence-graph-d30ee-6b65dbc99c-sjpgv namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 13ee2214-2cf8-46c3-985f-7335f1bc1aaf kind: Event lastTimestamp: "2026-07-10T19:21:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" already present on machine metadata: creationTimestamp: "2026-07-10T19:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:17Z" name: sequence-graph-d30ee-6b65dbc99c-sjpgv.18c103f4e786d24f namespace: kserve-ci-e2e-test resourceVersion: "21345" uid: 46dfc0cc-be02-49b8-aada-6118b4de35d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d30ee} kind: Pod name: sequence-graph-d30ee-6b65dbc99c-sjpgv namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 13ee2214-2cf8-46c3-985f-7335f1bc1aaf kind: Event lastTimestamp: "2026-07-10T19:21:17Z" message: 'Created container: sequence-graph-d30ee' metadata: creationTimestamp: "2026-07-10T19:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:17Z" name: sequence-graph-d30ee-6b65dbc99c-sjpgv.18c103f4ea86ff4d namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: d17f7405-f184-4bec-a9a2-f30e90a1fa3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d30ee} kind: Pod name: sequence-graph-d30ee-6b65dbc99c-sjpgv namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 13ee2214-2cf8-46c3-985f-7335f1bc1aaf kind: Event lastTimestamp: "2026-07-10T19:21:17Z" message: Started container sequence-graph-d30ee metadata: creationTimestamp: "2026-07-10T19:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:17Z" name: sequence-graph-d30ee-6b65dbc99c-sjpgv.18c103f4eaf28495 namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: 7c808fa0-5494-4936-97de-269ee58698c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d30ee} kind: Pod name: sequence-graph-d30ee-6b65dbc99c-sjpgv namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 13ee2214-2cf8-46c3-985f-7335f1bc1aaf kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: Stopping container sequence-graph-d30ee metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:27Z" name: sequence-graph-d30ee-6b65dbc99c-sjpgv.18c103f72bb756f8 namespace: kserve-ci-e2e-test resourceVersion: "21453" uid: ecf1cfd0-45a7-428c-bf45-074927a2e043 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:21:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d30ee} kind: Pod name: sequence-graph-d30ee-6b65dbc99c-sjpgv namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 13ee2214-2cf8-46c3-985f-7335f1bc1aaf kind: Event lastTimestamp: "2026-07-10T19:21:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:54Z" name: sequence-graph-d30ee-6b65dbc99c-sjpgv.18c103f7b456aa41 namespace: kserve-ci-e2e-test resourceVersion: "21819" uid: 64307539-3a75-410c-bb80-5a435f5c5521 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-d30ee-6b65dbc99c namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 498f318a-9104-4960-bedb-0d003cbbd751 kind: Event lastTimestamp: "2026-07-10T19:21:17Z" message: 'Created pod: sequence-graph-d30ee-6b65dbc99c-sjpgv' metadata: creationTimestamp: "2026-07-10T19:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:17Z" name: sequence-graph-d30ee-6b65dbc99c.18c103f4cc58c08a namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: 7912f711-a059-4b07-9a3a-91c6dc6449f7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-d30ee namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: 248a2890-e1aa-409c-9080-c132af31f77c kind: Event lastTimestamp: "2026-07-10T19:21:17Z" message: Scaled up replica set sequence-graph-d30ee-6b65dbc99c from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:17Z" name: sequence-graph-d30ee.18c103f4cb5afa3e namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: 9954003e-2ab9-4a0e-8944-64a8ee2c03ec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-d30ee namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: 93f723e9-b184-4a17-bcff-be72fa9e1a66 kind: Event lastTimestamp: "2026-07-10T19:21:17Z" message: InferenceGraph [sequence-graph-d30ee] is Ready metadata: creationTimestamp: "2026-07-10T19:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:17Z" name: sequence-graph-d30ee.18c103f4cfe3d698 namespace: kserve-ci-e2e-test resourceVersion: "21332" uid: d1b150c4-b1be-452c-9033-e275718e5ade reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-d30ee namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: 93f723e9-b184-4a17-bcff-be72fa9e1a66 kind: Event lastTimestamp: "2026-07-10T19:21:17Z" message: 'Failed to update status for InferenceGraph "sequence-graph-d30ee": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-d30ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:17Z" name: sequence-graph-d30ee.18c103f4d280ab83 namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: d622242b-7b90-4ca8-b721-6967c232fb09 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-d30ee namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: 93f723e9-b184-4a17-bcff-be72fa9e1a66 kind: Event lastTimestamp: "2026-07-10T19:21:17Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-d30ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:17Z" name: sequence-graph-d30ee.18c103f4d280f0ce namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: 3819a109-c509-457c-9551-e05baa516401 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:32:26.903167Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-1e1e7-5cd9fb55-x5p4n namespace: kserve-ci-e2e-test resourceVersion: "27280" uid: a935b160-e3a6-483d-85c8-55fd2500d0af kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-1e1e7-5cd9fb55-x5p4n to ip-10-0-131-123.ec2.internal metadata: creationTimestamp: "2026-07-10T19:32:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:32:26Z" name: splitter-graph-1e1e7-5cd9fb55-x5p4n.18c10490b0c8c846 namespace: kserve-ci-e2e-test resourceVersion: "27286" uid: eedbbd4b-c0e1-4a49-a328-7e23ab6b20e5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:27Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-1e1e7-5cd9fb55-x5p4n namespace: kserve-ci-e2e-test resourceVersion: "27289" uid: a935b160-e3a6-483d-85c8-55fd2500d0af kind: Event lastTimestamp: "2026-07-10T19:32:27Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:32:27Z" name: splitter-graph-1e1e7-5cd9fb55-x5p4n.18c10490c9e74ccc namespace: kserve-ci-e2e-test resourceVersion: "27304" uid: 73f4b27c-bc3d-44d0-af83-c74a9efba3fc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1e1e7} kind: Pod name: splitter-graph-1e1e7-5cd9fb55-x5p4n namespace: kserve-ci-e2e-test resourceVersion: "27283" uid: a935b160-e3a6-483d-85c8-55fd2500d0af kind: Event lastTimestamp: "2026-07-10T19:32:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" already present on machine metadata: creationTimestamp: "2026-07-10T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:32:27Z" name: splitter-graph-1e1e7-5cd9fb55-x5p4n.18c10490cb22a4d7 namespace: kserve-ci-e2e-test resourceVersion: "27306" uid: 921923a9-d65a-427f-99c6-90ff4672f423 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1e1e7} kind: Pod name: splitter-graph-1e1e7-5cd9fb55-x5p4n namespace: kserve-ci-e2e-test resourceVersion: "27283" uid: a935b160-e3a6-483d-85c8-55fd2500d0af kind: Event lastTimestamp: "2026-07-10T19:32:27Z" message: 'Created container: splitter-graph-1e1e7' metadata: creationTimestamp: "2026-07-10T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:32:27Z" name: splitter-graph-1e1e7-5cd9fb55-x5p4n.18c10490cdffc9b5 namespace: kserve-ci-e2e-test resourceVersion: "27307" uid: 752a340a-b55a-4cdd-bcb8-0a6c838b07f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1e1e7} kind: Pod name: splitter-graph-1e1e7-5cd9fb55-x5p4n namespace: kserve-ci-e2e-test resourceVersion: "27283" uid: a935b160-e3a6-483d-85c8-55fd2500d0af kind: Event lastTimestamp: "2026-07-10T19:32:27Z" message: Started container splitter-graph-1e1e7 metadata: creationTimestamp: "2026-07-10T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:32:27Z" name: splitter-graph-1e1e7-5cd9fb55-x5p4n.18c10490ce6e92ad namespace: kserve-ci-e2e-test resourceVersion: "27308" uid: 72f70776-3dd8-4573-b61f-3d4b23d53177 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:40:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1e1e7} kind: Pod name: splitter-graph-1e1e7-5cd9fb55-x5p4n namespace: kserve-ci-e2e-test resourceVersion: "27283" uid: a935b160-e3a6-483d-85c8-55fd2500d0af kind: Event lastTimestamp: "2026-07-10T19:40:41Z" message: Stopping container splitter-graph-1e1e7 metadata: creationTimestamp: "2026-07-10T19:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:40:41Z" name: splitter-graph-1e1e7-5cd9fb55-x5p4n.18c10503d6b69a12 namespace: kserve-ci-e2e-test resourceVersion: "30747" uid: 3f86680d-0121-4de5-82b2-cb8f782b5263 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:40:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1e1e7} kind: Pod name: splitter-graph-1e1e7-5cd9fb55-x5p4n namespace: kserve-ci-e2e-test resourceVersion: "27283" uid: a935b160-e3a6-483d-85c8-55fd2500d0af kind: Event lastTimestamp: "2026-07-10T19:41:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:41:08Z" name: splitter-graph-1e1e7-5cd9fb55-x5p4n.18c10504573b6393 namespace: kserve-ci-e2e-test resourceVersion: "30976" uid: 1856718b-fc5e-4d89-8e6a-deab8bc16543 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-1e1e7-5cd9fb55 namespace: kserve-ci-e2e-test resourceVersion: "27275" uid: 03edcfed-b535-46f5-aa70-c6fe89d3df65 kind: Event lastTimestamp: "2026-07-10T19:32:26Z" message: 'Created pod: splitter-graph-1e1e7-5cd9fb55-x5p4n' metadata: creationTimestamp: "2026-07-10T19:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:26Z" name: splitter-graph-1e1e7-5cd9fb55.18c10490b0374650 namespace: kserve-ci-e2e-test resourceVersion: "27282" uid: 5ef9644e-f4b9-41b8-b7ec-a01e4d29c0d8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "27272" uid: d6be5f22-6a03-4e46-9bdd-5b5063cf0cda kind: Event lastTimestamp: "2026-07-10T19:32:26Z" message: Scaled up replica set splitter-graph-1e1e7-5cd9fb55 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:26Z" name: splitter-graph-1e1e7.18c10490af31a7a2 namespace: kserve-ci-e2e-test resourceVersion: "27277" uid: 6771b2bc-5f0c-489b-903f-998317bbe473 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "27293" uid: bcd6810f-6974-4044-9d76-fec3b653b9dd kind: Event lastTimestamp: "2026-07-10T19:32:26Z" message: InferenceGraph [splitter-graph-1e1e7] is Ready metadata: creationTimestamp: "2026-07-10T19:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:32:26Z" name: splitter-graph-1e1e7.18c10490b45dc4dc namespace: kserve-ci-e2e-test resourceVersion: "27295" uid: 01186830-bbda-4923-a5f5-c53040bb22bd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "27265" uid: bcd6810f-6974-4044-9d76-fec3b653b9dd kind: Event lastTimestamp: "2026-07-10T19:32:27Z" message: 'Failed to update status for InferenceGraph "splitter-graph-1e1e7": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-1e1e7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:32:27Z" name: splitter-graph-1e1e7.18c10490b6f45ffe namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 8eabf2be-395a-426b-a91f-1af48d75711e reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "27265" uid: bcd6810f-6974-4044-9d76-fec3b653b9dd kind: Event lastTimestamp: "2026-07-10T19:32:27Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-1e1e7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:32:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:32:27Z" name: splitter-graph-1e1e7.18c10490b6f4b6d4 namespace: kserve-ci-e2e-test resourceVersion: "27300" uid: bd255f9e-cad2-4e64-97a3-2c821a5c749f reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "27273" uid: 4f7dadb1-abfa-47f0-9576-7cbf72e823d4 kind: Event lastTimestamp: "2026-07-10T19:32:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:41Z" name: splitter-graph-1e1e7.18c104942e280a37 namespace: kserve-ci-e2e-test resourceVersion: "27417" uid: ed7267d1-94d6-48bd-a852-a78534dd2f08 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "27273" uid: 4f7dadb1-abfa-47f0-9576-7cbf72e823d4 kind: Event lastTimestamp: "2026-07-10T19:32:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:41Z" name: splitter-graph-1e1e7.18c104942e8cff60 namespace: kserve-ci-e2e-test resourceVersion: "27420" uid: b0ea3af1-5f48-4c48-9207-e81f2091d36d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "27416" uid: 4f7dadb1-abfa-47f0-9576-7cbf72e823d4 kind: Event lastTimestamp: "2026-07-10T19:32:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:56Z" name: splitter-graph-1e1e7.18c10497ad408eaf namespace: kserve-ci-e2e-test resourceVersion: "27527" uid: e88653b2-f75f-4d46-a931-2412a8395ea2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "27416" uid: 4f7dadb1-abfa-47f0-9576-7cbf72e823d4 kind: Event lastTimestamp: "2026-07-10T19:32:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:56Z" name: splitter-graph-1e1e7.18c10497ad87560c namespace: kserve-ci-e2e-test resourceVersion: "27529" uid: d38fdf08-e61e-4dfc-9539-ea3cac1159f9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:31:16.556346Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4b29b-7b79559756-thmqw namespace: kserve-ci-e2e-test resourceVersion: "26469" uid: c41ebd47-f76f-488e-bd5b-6858d15638d9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-4b29b-7b79559756-thmqw to ip-10-0-131-123.ec2.internal metadata: creationTimestamp: "2026-07-10T19:31:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:31:16Z" name: splitter-graph-4b29b-7b79559756-thmqw.18c104804fc98070 namespace: kserve-ci-e2e-test resourceVersion: "26474" uid: 75eb0fb1-e2e5-417e-b772-ee9f63fb28b3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:16Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4b29b-7b79559756-thmqw namespace: kserve-ci-e2e-test resourceVersion: "26480" uid: c41ebd47-f76f-488e-bd5b-6858d15638d9 kind: Event lastTimestamp: "2026-07-10T19:31:16Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:31:16Z" name: splitter-graph-4b29b-7b79559756-thmqw.18c104806904d762 namespace: kserve-ci-e2e-test resourceVersion: "26491" uid: 0ea3e8cd-218d-467d-adaf-a45f816fe668 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4b29b} kind: Pod name: splitter-graph-4b29b-7b79559756-thmqw namespace: kserve-ci-e2e-test resourceVersion: "26470" uid: c41ebd47-f76f-488e-bd5b-6858d15638d9 kind: Event lastTimestamp: "2026-07-10T19:31:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" already present on machine metadata: creationTimestamp: "2026-07-10T19:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:17Z" name: splitter-graph-4b29b-7b79559756-thmqw.18c104806a42fd22 namespace: kserve-ci-e2e-test resourceVersion: "26493" uid: 9693002a-d693-41b4-bffb-a349edf7258a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4b29b} kind: Pod name: splitter-graph-4b29b-7b79559756-thmqw namespace: kserve-ci-e2e-test resourceVersion: "26470" uid: c41ebd47-f76f-488e-bd5b-6858d15638d9 kind: Event lastTimestamp: "2026-07-10T19:31:17Z" message: 'Created container: splitter-graph-4b29b' metadata: creationTimestamp: "2026-07-10T19:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:17Z" name: splitter-graph-4b29b-7b79559756-thmqw.18c104806d3d91ed namespace: kserve-ci-e2e-test resourceVersion: "26494" uid: 855ab373-2072-46f9-a62a-237a6acf210e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4b29b} kind: Pod name: splitter-graph-4b29b-7b79559756-thmqw namespace: kserve-ci-e2e-test resourceVersion: "26470" uid: c41ebd47-f76f-488e-bd5b-6858d15638d9 kind: Event lastTimestamp: "2026-07-10T19:31:17Z" message: Started container splitter-graph-4b29b metadata: creationTimestamp: "2026-07-10T19:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:17Z" name: splitter-graph-4b29b-7b79559756-thmqw.18c104806da815c9 namespace: kserve-ci-e2e-test resourceVersion: "26495" uid: 0ca7dff2-d6f0-44ba-961a-5053fd26ee83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4b29b} kind: Pod name: splitter-graph-4b29b-7b79559756-thmqw namespace: kserve-ci-e2e-test resourceVersion: "26470" uid: c41ebd47-f76f-488e-bd5b-6858d15638d9 kind: Event lastTimestamp: "2026-07-10T19:31:26Z" message: Stopping container splitter-graph-4b29b metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:26Z" name: splitter-graph-4b29b-7b79559756-thmqw.18c10482a95a9b0a namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: c9c82899-1ed7-4646-861f-2ae28fc0988e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4b29b} kind: Pod name: splitter-graph-4b29b-7b79559756-thmqw namespace: kserve-ci-e2e-test resourceVersion: "26470" uid: c41ebd47-f76f-488e-bd5b-6858d15638d9 kind: Event lastTimestamp: "2026-07-10T19:31:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:53Z" name: splitter-graph-4b29b-7b79559756-thmqw.18c104830cd7d1cd namespace: kserve-ci-e2e-test resourceVersion: "26995" uid: f17eebb3-2c5b-4ddc-bc47-459e65f41e33 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-4b29b-7b79559756 namespace: kserve-ci-e2e-test resourceVersion: "26466" uid: 605b5d09-c884-4700-ba2f-b8da7fe16a74 kind: Event lastTimestamp: "2026-07-10T19:31:16Z" message: 'Created pod: splitter-graph-4b29b-7b79559756-thmqw' metadata: creationTimestamp: "2026-07-10T19:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:16Z" name: splitter-graph-4b29b-7b79559756.18c104804f5301d1 namespace: kserve-ci-e2e-test resourceVersion: "26471" uid: 5a46a170-d92f-45fe-8e68-354450883af0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-4b29b namespace: kserve-ci-e2e-test resourceVersion: "26463" uid: b36a323d-0b76-4c29-8a0d-0f7fce85993f kind: Event lastTimestamp: "2026-07-10T19:31:16Z" message: Scaled up replica set splitter-graph-4b29b-7b79559756 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:16Z" name: splitter-graph-4b29b.18c104804e83f5fa namespace: kserve-ci-e2e-test resourceVersion: "26467" uid: f687d5fc-3a44-472f-92b1-069b195bd4a2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-4b29b namespace: kserve-ci-e2e-test resourceVersion: "26483" uid: 7a9aaa19-2141-4a67-a211-bbbd44495c8e kind: Event lastTimestamp: "2026-07-10T19:31:16Z" message: InferenceGraph [splitter-graph-4b29b] is Ready metadata: creationTimestamp: "2026-07-10T19:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:16Z" name: splitter-graph-4b29b.18c10480538d4fb3 namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 641a5080-0a2c-4050-9f0b-b48449243a2b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:30:42.573447Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr namespace: kserve-ci-e2e-test resourceVersion: "26087" uid: 5992cd9f-e127-4fe4-80d1-6257a5b4e44c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:30:42Z" name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr.18c10478663f8870 namespace: kserve-ci-e2e-test resourceVersion: "26092" uid: 267aaf71-9982-48b2-a66b-a8dea24f6ed4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr namespace: kserve-ci-e2e-test resourceVersion: "26098" uid: 5992cd9f-e127-4fe4-80d1-6257a5b4e44c kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:30:43Z" name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr.18c104788071fef9 namespace: kserve-ci-e2e-test resourceVersion: "26136" uid: 15f21de2-651e-40fa-9fe0-a89828f18809 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 5992cd9f-e127-4fe4-80d1-6257a5b4e44c kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:43Z" name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr.18c1047881f4380a namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 3dd76fec-030e-4cec-9d31-4683bd5b5e7e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 5992cd9f-e127-4fe4-80d1-6257a5b4e44c kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:43Z" name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr.18c1047884cd9d7d namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: 2fde2314-a7ae-4596-8b1d-80b06e8717ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 5992cd9f-e127-4fe4-80d1-6257a5b4e44c kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:43Z" name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr.18c10478853df5da namespace: kserve-ci-e2e-test resourceVersion: "26141" uid: c6dbf5a0-c673-4c5d-b25e-ff7095cce0d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 5992cd9f-e127-4fe4-80d1-6257a5b4e44c kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:43Z" name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr.18c104788548742e namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: 35603e36-eb6f-4c8e-867a-b9105e087c0a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 5992cd9f-e127-4fe4-80d1-6257a5b4e44c kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:43Z" name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr.18c1047888632f27 namespace: kserve-ci-e2e-test resourceVersion: "26143" uid: 02c86c2b-b5bb-44c2-83db-7703589f095a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 5992cd9f-e127-4fe4-80d1-6257a5b4e44c kind: Event lastTimestamp: "2026-07-10T19:30:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:43Z" name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr.18c1047888d895c1 namespace: kserve-ci-e2e-test resourceVersion: "26144" uid: 28ed4681-1d4f-405e-977f-0dbac7de56b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:30:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 5992cd9f-e127-4fe4-80d1-6257a5b4e44c kind: Event lastTimestamp: "2026-07-10T19:31:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:30:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:19Z" name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr.18c10478ae637b6a namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: d4974f50-fbb4-4c28-b733-80ad70036be4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 5992cd9f-e127-4fe4-80d1-6257a5b4e44c kind: Event lastTimestamp: "2026-07-10T19:48:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:48:01Z" name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr.18c1056a658fa868 namespace: kserve-ci-e2e-test resourceVersion: "33827" uid: 83c56fed-bfec-46fd-bcb7-961f565ebd87 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 5992cd9f-e127-4fe4-80d1-6257a5b4e44c kind: Event lastTimestamp: "2026-07-10T19:48:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:48:01Z" name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr.18c1056a65902823 namespace: kserve-ci-e2e-test resourceVersion: "33829" uid: fa9ad771-9d28-43fa-a061-d918d21c258f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:48:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 5992cd9f-e127-4fe4-80d1-6257a5b4e44c kind: Event lastTimestamp: "2026-07-10T19:48:04Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:48:04Z" name: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr.18c1056b0f3dcaf7 namespace: kserve-ci-e2e-test resourceVersion: "33896" uid: f8ff76d0-cd1f-4889-8f08-c8ead19f113f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1c787-predictor-5b4bb49b7 namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: 266afad8-e668-4fdc-9f29-134cdcd5a0ba kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: 'Created pod: success-200-isvc-1c787-predictor-5b4bb49b7-8qlrr' metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:30:42Z" name: success-200-isvc-1c787-predictor-5b4bb49b7.18c1047865a53093 namespace: kserve-ci-e2e-test resourceVersion: "26091" uid: 272fa11f-4b07-4531-b1e0-d9629aae5ccf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1c787-predictor namespace: kserve-ci-e2e-test resourceVersion: "26068" uid: 5acf28bd-90a4-4d5f-9b4f-a52854c969b9 kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: Scaled up replica set success-200-isvc-1c787-predictor-5b4bb49b7 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:30:42Z" name: success-200-isvc-1c787-predictor.18c1047863512d7b namespace: kserve-ci-e2e-test resourceVersion: "26077" uid: 4c92afe8-0c0e-4837-8abb-9379f1de04e9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:30:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1c787-predictor namespace: kserve-ci-e2e-test resourceVersion: "26070" uid: 66008bac-b0a4-4cfe-90bc-329eba2ea28b kind: Event lastTimestamp: "2026-07-10T19:31:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:12Z" name: success-200-isvc-1c787-predictor.18c1047be1bc1768 namespace: kserve-ci-e2e-test resourceVersion: "26424" uid: da9be0f4-7164-44c5-a325-7d6728129728 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:30:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1c787-predictor namespace: kserve-ci-e2e-test resourceVersion: "26070" uid: 66008bac-b0a4-4cfe-90bc-329eba2ea28b kind: Event lastTimestamp: "2026-07-10T19:31:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:12Z" name: success-200-isvc-1c787-predictor.18c1047be2222263 namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: c323d72c-4053-4582-a571-dacf794f594e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:31:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1c787-predictor namespace: kserve-ci-e2e-test resourceVersion: "26288" uid: 66008bac-b0a4-4cfe-90bc-329eba2ea28b kind: Event lastTimestamp: "2026-07-10T19:31:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:42Z" name: success-200-isvc-1c787-predictor.18c10482e01b3aae namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: 9ebcad8d-acb5-4f0f-a87d-f54b56f65745 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:31:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1c787-predictor namespace: kserve-ci-e2e-test resourceVersion: "26288" uid: 66008bac-b0a4-4cfe-90bc-329eba2ea28b kind: Event lastTimestamp: "2026-07-10T19:31:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:42Z" name: success-200-isvc-1c787-predictor.18c10482e0e2fe17 namespace: kserve-ci-e2e-test resourceVersion: "26880" uid: c8b1f915-6125-4584-b3ad-ef366af56919 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1c787 namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 621c2c7c-8d2a-4a5f-993b-4131025fa936 kind: Event lastTimestamp: "2026-07-10T19:31:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:29Z" name: success-200-isvc-1c787.18c1047863639720 namespace: kserve-ci-e2e-test resourceVersion: "26755" uid: d651d447-f83a-476e-a4ea-89fd3ae84164 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1c787 namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 621c2c7c-8d2a-4a5f-993b-4131025fa936 kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: 'Failed to update status for InferenceService "success-200-isvc-1c787": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1c787": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:30:42Z" name: success-200-isvc-1c787.18c1047863e2924f namespace: kserve-ci-e2e-test resourceVersion: "26081" uid: 79d3fca8-1b6c-43ab-afc5-3eda8a29674c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1c787 namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 621c2c7c-8d2a-4a5f-993b-4131025fa936 kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1c787": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:30:42Z" name: success-200-isvc-1c787.18c1047863e2e894 namespace: kserve-ci-e2e-test resourceVersion: "26082" uid: 8310d18b-a058-4d55-bf82-c2d8a53b9798 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1c787 namespace: kserve-ci-e2e-test resourceVersion: "26753" uid: 621c2c7c-8d2a-4a5f-993b-4131025fa936 kind: Event lastTimestamp: "2026-07-10T19:31:29Z" message: InferenceService [success-200-isvc-1c787] is Ready metadata: creationTimestamp: "2026-07-10T19:31:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:29Z" name: success-200-isvc-1c787.18c104836986bd34 namespace: kserve-ci-e2e-test resourceVersion: "26754" uid: b3167813-5dac-4124-830e-49359f711910 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:31:26.832689Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49 namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 4c4134c2-98cf-4853-b50d-30ac551f5683 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49 to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:31:26Z" name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49.18c10482b44dff62 namespace: kserve-ci-e2e-test resourceVersion: "26649" uid: 76afd791-eb9b-4872-84e9-23db8b80c005 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: 4c4134c2-98cf-4853-b50d-30ac551f5683 kind: Event lastTimestamp: "2026-07-10T19:31:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1e1e7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:27Z" name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49.18c10482bebabc4b namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: 9fb0d743-bd6f-484c-b4b1-f5b604ba96d3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49 namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: 4c4134c2-98cf-4853-b50d-30ac551f5683 kind: Event lastTimestamp: "2026-07-10T19:31:27Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:31:27Z" name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49.18c10482f220c472 namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: fb132111-3529-4011-b1be-707cc6314f98 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: 4c4134c2-98cf-4853-b50d-30ac551f5683 kind: Event lastTimestamp: "2026-07-10T19:31:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:27Z" name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49.18c10482f3a2ae1a namespace: kserve-ci-e2e-test resourceVersion: "26713" uid: 87408cd9-8880-4eee-bd38-7cdfad0e230c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: 4c4134c2-98cf-4853-b50d-30ac551f5683 kind: Event lastTimestamp: "2026-07-10T19:31:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:27Z" name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49.18c10482f7a0e5b4 namespace: kserve-ci-e2e-test resourceVersion: "26714" uid: 9a168ed5-f5a6-47da-be46-23ab31def56d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: 4c4134c2-98cf-4853-b50d-30ac551f5683 kind: Event lastTimestamp: "2026-07-10T19:31:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:27Z" name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49.18c10482f82405bf namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 92ece07d-9019-4012-8044-2e7c7cfc7502 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: 4c4134c2-98cf-4853-b50d-30ac551f5683 kind: Event lastTimestamp: "2026-07-10T19:31:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:27Z" name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49.18c10482f830ce4a namespace: kserve-ci-e2e-test resourceVersion: "26716" uid: bae88d3d-fdcc-4c43-9d2b-68df12711c7d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: 4c4134c2-98cf-4853-b50d-30ac551f5683 kind: Event lastTimestamp: "2026-07-10T19:31:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:28Z" name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49.18c10482fbbb2622 namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: f24beabb-af49-4277-985b-61d2f2d23dec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: 4c4134c2-98cf-4853-b50d-30ac551f5683 kind: Event lastTimestamp: "2026-07-10T19:31:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:28Z" name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49.18c10482fc42b0dc namespace: kserve-ci-e2e-test resourceVersion: "26721" uid: fb356a5f-0cac-4b31-b8de-1ca609034d6b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:31:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: 4c4134c2-98cf-4853-b50d-30ac551f5683 kind: Event lastTimestamp: "2026-07-10T19:32:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:31:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:32:05Z" name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49.18c104833465fbef namespace: kserve-ci-e2e-test resourceVersion: "27090" uid: 695904a1-ff30-482c-b26f-083953c83c5d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:40:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: 4c4134c2-98cf-4853-b50d-30ac551f5683 kind: Event lastTimestamp: "2026-07-10T19:40:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:40:41Z" name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49.18c10503de12b817 namespace: kserve-ci-e2e-test resourceVersion: "30763" uid: 509ccafa-435e-4f40-b851-b76c10180c63 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:40:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: 4c4134c2-98cf-4853-b50d-30ac551f5683 kind: Event lastTimestamp: "2026-07-10T19:40:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:40:41Z" name: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49.18c10503de13875c namespace: kserve-ci-e2e-test resourceVersion: "30764" uid: 0197120b-1fec-479f-8a23-80be179b64fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1e1e7-predictor-b8d4c67bf namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: b8fddfa8-f585-4aaa-87f6-c387d1e6c0f8 kind: Event lastTimestamp: "2026-07-10T19:31:26Z" message: 'Created pod: success-200-isvc-1e1e7-predictor-b8d4c67bf-drq49' metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:26Z" name: success-200-isvc-1e1e7-predictor-b8d4c67bf.18c10482b3bdc6ea namespace: kserve-ci-e2e-test resourceVersion: "26645" uid: 5f7fdd82-e168-4d1f-8428-1c5611168b35 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1e1e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "26625" uid: 7f09bb73-e88d-453f-be4f-80b53a1cd661 kind: Event lastTimestamp: "2026-07-10T19:31:26Z" message: Scaled up replica set success-200-isvc-1e1e7-predictor-b8d4c67bf from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:26Z" name: success-200-isvc-1e1e7-predictor.18c10482b2133277 namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: 28665ae2-5de8-44e9-a944-2da610d49f48 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1e1e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: fc901fee-e854-48da-ab5f-76d3228da3cf kind: Event lastTimestamp: "2026-07-10T19:32:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:11Z" name: success-200-isvc-1e1e7-predictor.18c1048631163d57 namespace: kserve-ci-e2e-test resourceVersion: "27135" uid: f780cb0e-6ed7-4ba9-a591-94296f0aff04 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1e1e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: fc901fee-e854-48da-ab5f-76d3228da3cf kind: Event lastTimestamp: "2026-07-10T19:32:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:11Z" name: success-200-isvc-1e1e7-predictor.18c10486317bc6ab namespace: kserve-ci-e2e-test resourceVersion: "27136" uid: 9c829ce2-fc7d-4103-8e04-dd59de3ff89c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:31:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1e1e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "26857" uid: fc901fee-e854-48da-ab5f-76d3228da3cf kind: Event lastTimestamp: "2026-07-10T19:32:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:31:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:41Z" name: success-200-isvc-1e1e7-predictor.18c10489b01c42f5 namespace: kserve-ci-e2e-test resourceVersion: "27418" uid: d5fc504d-e7b4-4905-9877-4f4e7438f216 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:31:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1e1e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "26857" uid: fc901fee-e854-48da-ab5f-76d3228da3cf kind: Event lastTimestamp: "2026-07-10T19:32:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:31:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:41Z" name: success-200-isvc-1e1e7-predictor.18c10489b079ae56 namespace: kserve-ci-e2e-test resourceVersion: "27419" uid: 9303788c-0e16-414d-bd8e-1a92fea4f4dd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "26598" uid: 216b4c97-0783-48c0-bf8d-2ffe16c5af8b kind: Event lastTimestamp: "2026-07-10T19:32:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:32:15Z" name: success-200-isvc-1e1e7.18c10482b28ba3be namespace: kserve-ci-e2e-test resourceVersion: "27185" uid: fb4dc925-dfd9-4f82-93ec-fbe1c0659eb7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "26598" uid: 216b4c97-0783-48c0-bf8d-2ffe16c5af8b kind: Event lastTimestamp: "2026-07-10T19:31:26Z" message: 'Failed to update status for InferenceService "success-200-isvc-1e1e7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1e1e7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:26Z" name: success-200-isvc-1e1e7.18c10482b2e3419f namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: d9332da0-6740-4d60-827d-1be0e35682fb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "26598" uid: 216b4c97-0783-48c0-bf8d-2ffe16c5af8b kind: Event lastTimestamp: "2026-07-10T19:31:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1e1e7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:26Z" name: success-200-isvc-1e1e7.18c10482b2e38d6f namespace: kserve-ci-e2e-test resourceVersion: "26642" uid: c97bacf7-02bd-4a1f-9214-20cdeadf5eaa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:32:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1e1e7 namespace: kserve-ci-e2e-test resourceVersion: "27178" uid: 216b4c97-0783-48c0-bf8d-2ffe16c5af8b kind: Event lastTimestamp: "2026-07-10T19:32:15Z" message: InferenceService [success-200-isvc-1e1e7] is Ready metadata: creationTimestamp: "2026-07-10T19:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:32:15Z" name: success-200-isvc-1e1e7.18c1048df10da4d9 namespace: kserve-ci-e2e-test resourceVersion: "27181" uid: 96c1333c-134a-4d7b-a3a6-1cb13c4478db reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:11:02.301702Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52 namespace: kserve-ci-e2e-test resourceVersion: "16219" uid: 0d6e9d06-c511-4bfe-9373-701578db33e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-31f26-predictor-7599c98b7f-ctz52 to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:11:02Z" name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52.18c103659891fc9c namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 2b05aa41-b947-4633-b460-540d935db95c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52 namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: 0d6e9d06-c511-4bfe-9373-701578db33e5 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:11:02Z" name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52.18c10365b295172e namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 7ffe7ca5-7ae6-453d-b16e-5e85ee584d53 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 0d6e9d06-c511-4bfe-9373-701578db33e5 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52.18c10365b4194eef namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 6c0b7977-1b23-4872-94b5-3ecd7f6d055f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 0d6e9d06-c511-4bfe-9373-701578db33e5 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52.18c10365b746920b namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: 1f3c914c-c6c4-4171-9bdd-07cf3a0894f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 0d6e9d06-c511-4bfe-9373-701578db33e5 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52.18c10365b7cd1ff2 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: dd0fa242-673e-4172-adc6-a137215f5aaa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 0d6e9d06-c511-4bfe-9373-701578db33e5 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52.18c10365b7d8388b namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: d3740abd-c3b0-467c-b514-81f0acb8469d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 0d6e9d06-c511-4bfe-9373-701578db33e5 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52.18c10365bb8f00fe namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: d46d3dd9-4534-4a5e-aa65-232bed4bad06 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 0d6e9d06-c511-4bfe-9373-701578db33e5 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:02Z" name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52.18c10365bc2b6d9c namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: 1735a302-7636-4220-9421-88ea335c278e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-10T19:11:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 0d6e9d06-c511-4bfe-9373-701578db33e5 kind: Event lastTimestamp: "2026-07-10T19:20:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:20Z" name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52.18c103660977a75e namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: 7bf2d61a-686a-4cdb-8182-d6584c2d74f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 0d6e9d06-c511-4bfe-9373-701578db33e5 kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:17Z" name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52.18c103e6cedea4cf namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 4bdde79e-c13d-45ba-a3fc-236c0676727d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 0d6e9d06-c511-4bfe-9373-701578db33e5 kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:17Z" name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52.18c103e6cedf233f namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: e7495a4f-50ea-49a1-8077-108c47b51392 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 0d6e9d06-c511-4bfe-9373-701578db33e5 kind: Event lastTimestamp: "2026-07-10T19:20:20Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:20Z" name: success-200-isvc-31f26-predictor-7599c98b7f-ctz52.18c103e77de65cd5 namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: b70ee05b-d77c-4401-acfd-82cec1efee70 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-31f26-predictor-7599c98b7f namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: e0004d37-bb19-496e-9ee0-e77b347b4af9 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: 'Created pod: success-200-isvc-31f26-predictor-7599c98b7f-ctz52' metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:02Z" name: success-200-isvc-31f26-predictor-7599c98b7f.18c1036597979dd7 namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: 51ebbd56-6c82-46cf-9bd5-abf2438f3e20 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-31f26-predictor namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: 959c09b0-8b9c-4af5-9bd6-bbe73e1ccb41 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: Scaled up replica set success-200-isvc-31f26-predictor-7599c98b7f from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:02Z" name: success-200-isvc-31f26-predictor.18c10365957e5096 namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: 7285c84f-5d69-43b1-97ff-5395889ab4b1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:11:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-31f26-predictor namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: 8b1ee946-9532-4111-a0ce-8123666b396f kind: Event lastTimestamp: "2026-07-10T19:11:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:32Z" name: success-200-isvc-31f26-predictor.18c1036914187db3 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: 56704490-5780-40c8-97c4-4becb97e1d5f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:11:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-31f26-predictor namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: 8b1ee946-9532-4111-a0ce-8123666b396f kind: Event lastTimestamp: "2026-07-10T19:11:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:32Z" name: success-200-isvc-31f26-predictor.18c103691489d0d0 namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: bfc4aed0-b099-4693-90c5-3af4bd669323 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:11:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-31f26-predictor namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: 8b1ee946-9532-4111-a0ce-8123666b396f kind: Event lastTimestamp: "2026-07-10T19:12:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:17Z" name: success-200-isvc-31f26-predictor.18c1037011f74c71 namespace: kserve-ci-e2e-test resourceVersion: "17008" uid: 8fd7a953-8f15-4971-8ffb-925d1ad42e78 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:11:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-31f26-predictor namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: 8b1ee946-9532-4111-a0ce-8123666b396f kind: Event lastTimestamp: "2026-07-10T19:12:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:17Z" name: success-200-isvc-31f26-predictor.18c10370125c5fd8 namespace: kserve-ci-e2e-test resourceVersion: "17009" uid: ac4886ae-8160-4ea5-b8ee-b17d7ae7eb4b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: 899753b7-c35a-4460-8129-39b6c1d1dde2 kind: Event lastTimestamp: "2026-07-10T19:11:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:50Z" name: success-200-isvc-31f26.18c1036595838902 namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: a03c6ce6-c07a-46cf-a792-dda7b2b2c3ff reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: 899753b7-c35a-4460-8129-39b6c1d1dde2 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: 'Failed to update status for InferenceService "success-200-isvc-31f26": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-31f26": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:02Z" name: success-200-isvc-31f26.18c10365960d8e16 namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: 60dbc184-3d4f-4caa-8eb3-1073ad7ef0cf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: 899753b7-c35a-4460-8129-39b6c1d1dde2 kind: Event lastTimestamp: "2026-07-10T19:11:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-31f26": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:02Z" name: success-200-isvc-31f26.18c10365960df834 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 596b052a-f2f3-49b9-9f41-b2f2abe6d0c1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-31f26 namespace: kserve-ci-e2e-test resourceVersion: "16760" uid: 899753b7-c35a-4460-8129-39b6c1d1dde2 kind: Event lastTimestamp: "2026-07-10T19:11:50Z" message: InferenceService [success-200-isvc-31f26] is Ready metadata: creationTimestamp: "2026-07-10T19:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:50Z" name: success-200-isvc-31f26.18c10370c4593e4a namespace: kserve-ci-e2e-test resourceVersion: "16761" uid: 9e539fb9-e2b3-4626-a2a8-67500f5428ea reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:19:41.272057Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 namespace: kserve-ci-e2e-test resourceVersion: "20104" uid: 0a8e24b7-7139-43b6-90b9-20860369b7bf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:19:41Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4.18c103de6d9d23d3 namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: 97de2d0f-ebf1-4e8e-a9c1-39163a5836ae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 0a8e24b7-7139-43b6-90b9-20860369b7bf kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-3dba1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:41Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4.18c103de77efd79f namespace: kserve-ci-e2e-test resourceVersion: "20144" uid: b26e03c9-003f-4318-8fda-947911cad01a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: 0a8e24b7-7139-43b6-90b9-20860369b7bf kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:19:42Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4.18c103deac375e5f namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: 5c3297d0-b1a9-44fa-b860-070aac8d3c02 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 0a8e24b7-7139-43b6-90b9-20860369b7bf kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:42Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4.18c103deadb3937c namespace: kserve-ci-e2e-test resourceVersion: "20161" uid: bd022b74-4c5b-4868-948a-c8f356f8418d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 0a8e24b7-7139-43b6-90b9-20860369b7bf kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:42Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4.18c103deb08bed57 namespace: kserve-ci-e2e-test resourceVersion: "20164" uid: 60adb8c5-2b66-41b4-96b1-8695c344b662 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 0a8e24b7-7139-43b6-90b9-20860369b7bf kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:42Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4.18c103deb162f7f0 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 4c40c9db-5f73-4b5d-a268-ec1917efc205 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 0a8e24b7-7139-43b6-90b9-20860369b7bf kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:42Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4.18c103deb16de480 namespace: kserve-ci-e2e-test resourceVersion: "20167" uid: 6765806d-610e-434a-80b3-229203c703bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 0a8e24b7-7139-43b6-90b9-20860369b7bf kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:42Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4.18c103deb44f7f2b namespace: kserve-ci-e2e-test resourceVersion: "20169" uid: 1eb3cb9e-7d83-4f06-9ea7-b132d15e0d49 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 0a8e24b7-7139-43b6-90b9-20860369b7bf kind: Event lastTimestamp: "2026-07-10T19:19:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:42Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4.18c103deb4cde039 namespace: kserve-ci-e2e-test resourceVersion: "20172" uid: 8a5958a0-f296-479e-ac1b-7ce549e2a3a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:19:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 0a8e24b7-7139-43b6-90b9-20860369b7bf kind: Event lastTimestamp: "2026-07-10T19:20:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:19:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:19Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4.18c103deddcbcead namespace: kserve-ci-e2e-test resourceVersion: "20605" uid: 8d32cb4c-e803-413a-9a15-8c1d0b695da7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 0a8e24b7-7139-43b6-90b9-20860369b7bf kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:51Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4.18c103eec84cd5cc namespace: kserve-ci-e2e-test resourceVersion: "20991" uid: b8ca9307-b61a-4a0b-905d-11df9c0610f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 0a8e24b7-7139-43b6-90b9-20860369b7bf kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:51Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4.18c103eec84d56af namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: 0c5e1ac4-1710-4c7e-9ae0-359e609c9e65 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 0a8e24b7-7139-43b6-90b9-20860369b7bf kind: Event lastTimestamp: "2026-07-10T19:20:54Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:54Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4.18c103ef65eebcc6 namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: 7ed7c831-72be-4c56-a163-a23f07f3bc9a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3dba1-predictor-6f64fbb6b5 namespace: kserve-ci-e2e-test resourceVersion: "20091" uid: da2dfe06-d3dd-4260-9200-af92db4fcc5d kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: 'Created pod: success-200-isvc-3dba1-predictor-6f64fbb6b5-cdmf4' metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:19:41Z" name: success-200-isvc-3dba1-predictor-6f64fbb6b5.18c103de6d0ced1e namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: ffdbd3ba-b70b-4291-ad57-461276b3016d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3dba1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20086" uid: a1706398-0c2f-43aa-98ab-95433343c043 kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: Scaled up replica set success-200-isvc-3dba1-predictor-6f64fbb6b5 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:19:41Z" name: success-200-isvc-3dba1-predictor.18c103de69918923 namespace: kserve-ci-e2e-test resourceVersion: "20093" uid: 77ee4a9e-848a-4597-a2f1-307b8713dec2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:19:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3dba1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: dead4055-12ad-4643-a767-eda9fc47443b kind: Event lastTimestamp: "2026-07-10T19:20:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:26Z" name: success-200-isvc-3dba1-predictor.18c103e1e8103f96 namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 5588ef27-0a10-4434-ba61-a187b32e057d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:19:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3dba1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: dead4055-12ad-4643-a767-eda9fc47443b kind: Event lastTimestamp: "2026-07-10T19:20:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:26Z" name: success-200-isvc-3dba1-predictor.18c103e1e871cd5b namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 88baa98f-58e4-4dc2-b126-58ded5b0a581 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3dba1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20295" uid: dead4055-12ad-4643-a767-eda9fc47443b kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:41Z" name: success-200-isvc-3dba1-predictor.18c103ec651ddb66 namespace: kserve-ci-e2e-test resourceVersion: "20817" uid: 9cf70dfa-6a8a-4022-9525-ee6ec539d1bd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3dba1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20295" uid: dead4055-12ad-4643-a767-eda9fc47443b kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:41Z" name: success-200-isvc-3dba1-predictor.18c103ec65813b45 namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: 6ed10a83-91f0-4191-a435-0ba3c9914a49 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3dba1 namespace: kserve-ci-e2e-test resourceVersion: "20065" uid: f9c42aa2-14d5-4650-8946-f04ea949b7b4 kind: Event lastTimestamp: "2026-07-10T19:20:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:29Z" name: success-200-isvc-3dba1.18c103de69cd12be namespace: kserve-ci-e2e-test resourceVersion: "20707" uid: 00ceac74-d634-4546-8521-634f2e8082b5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3dba1 namespace: kserve-ci-e2e-test resourceVersion: "20065" uid: f9c42aa2-14d5-4650-8946-f04ea949b7b4 kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-3dba1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3dba1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:19:41Z" name: success-200-isvc-3dba1.18c103de6a1cd59e namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: e0033b13-71ab-4848-b378-247762c99453 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3dba1 namespace: kserve-ci-e2e-test resourceVersion: "20065" uid: f9c42aa2-14d5-4650-8946-f04ea949b7b4 kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3dba1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:19:41Z" name: success-200-isvc-3dba1.18c103de6a1d1b4f namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: 9db247e8-083e-4a3a-9886-a87a679f26a2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3dba1 namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: f9c42aa2-14d5-4650-8946-f04ea949b7b4 kind: Event lastTimestamp: "2026-07-10T19:20:29Z" message: InferenceService [success-200-isvc-3dba1] is Ready metadata: creationTimestamp: "2026-07-10T19:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:29Z" name: success-200-isvc-3dba1.18c103e998b9dd9c namespace: kserve-ci-e2e-test resourceVersion: "20705" uid: 48d7427b-a73b-4fbf-8734-871cb00bc9fc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:10:26.356276Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: 68099bf1-5bb6-470a-b8f1-5e3a611f0679 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4aae5-predictor-f8cd6867f-plzsm to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm.18c1035d3a0e5b3a namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 658e8173-dd36-4998-b532-bcb06386a165 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 68099bf1-5bb6-470a-b8f1-5e3a611f0679 kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm.18c1035d54138427 namespace: kserve-ci-e2e-test resourceVersion: "15754" uid: 572971e0-46d3-48c1-bf99-16e61d1d6511 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 68099bf1-5bb6-470a-b8f1-5e3a611f0679 kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm.18c1035d55ac4223 namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: 87236767-2b96-4b83-a02c-daed851cf119 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 68099bf1-5bb6-470a-b8f1-5e3a611f0679 kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm.18c1035d58af24cd namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 858a243d-7230-4d27-abfa-cca5bbe90e58 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 68099bf1-5bb6-470a-b8f1-5e3a611f0679 kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm.18c1035d5923b021 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: 9be5f9e8-b3e5-41d7-a0ac-88aea30f084a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 68099bf1-5bb6-470a-b8f1-5e3a611f0679 kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm.18c1035d592d2722 namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: 9745d7bd-8026-4970-8ef1-59651db225f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 68099bf1-5bb6-470a-b8f1-5e3a611f0679 kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm.18c1035d5c54ac2e namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 43c84752-f291-4420-9696-692c19dc6aa4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 68099bf1-5bb6-470a-b8f1-5e3a611f0679 kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm.18c1035d5ccb0975 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: cf2a199f-ebbd-4a0b-8c21-f0c51849f8bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-10T19:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 68099bf1-5bb6-470a-b8f1-5e3a611f0679 kind: Event lastTimestamp: "2026-07-10T19:19:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:43Z" name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm.18c1035d62a9f0eb namespace: kserve-ci-e2e-test resourceVersion: "20182" uid: eaa404f3-fd8c-46c6-abce-d42013f127e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 68099bf1-5bb6-470a-b8f1-5e3a611f0679 kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:41Z" name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm.18c103de66897afb namespace: kserve-ci-e2e-test resourceVersion: "20067" uid: 1f8a4f13-262b-42a9-b917-48458c19d0c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 68099bf1-5bb6-470a-b8f1-5e3a611f0679 kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:41Z" name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm.18c103de6689f4fc namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: ecdf53fe-4f11-47bf-a83a-cbb1958188d7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 68099bf1-5bb6-470a-b8f1-5e3a611f0679 kind: Event lastTimestamp: "2026-07-10T19:19:43Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:19:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:43Z" name: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm.18c103ded745c614 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 83ba50a0-8a00-4a51-a19c-48a38dc43a20 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4aae5-predictor-f8cd6867f namespace: kserve-ci-e2e-test resourceVersion: "15694" uid: 5ec76072-3cfa-4ebb-a8e4-4dd0c3de8414 kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: 'Created pod: success-200-isvc-4aae5-predictor-f8cd6867f-plzsm' metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-4aae5-predictor-f8cd6867f.18c1035d396a9d42 namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: 11256f8f-3ca4-4650-bf9d-93d99eab07bd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4aae5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: 401d353c-d94e-4837-b570-d26527b414cd kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: Scaled up replica set success-200-isvc-4aae5-predictor-f8cd6867f from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-4aae5-predictor.18c1035d35ac3612 namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: cac868de-55f5-4dcd-85fb-7bac44a50bab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:10:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4aae5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: 4ab5a485-13aa-4d51-ae9e-bf8ed4f22b73 kind: Event lastTimestamp: "2026-07-10T19:11:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:11Z" name: success-200-isvc-4aae5-predictor.18c10360b49e69e5 namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: ea383bb5-fe5a-4346-953f-e2582fd69528 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:10:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4aae5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: 4ab5a485-13aa-4d51-ae9e-bf8ed4f22b73 kind: Event lastTimestamp: "2026-07-10T19:11:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:11Z" name: success-200-isvc-4aae5-predictor.18c10360b4fc93fd namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: a0591c37-fa0c-46dc-b13f-c1800e91d805 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:10:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4aae5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: 4ab5a485-13aa-4d51-ae9e-bf8ed4f22b73 kind: Event lastTimestamp: "2026-07-10T19:11:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:10:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:41Z" name: success-200-isvc-4aae5-predictor.18c1036433fc0c85 namespace: kserve-ci-e2e-test resourceVersion: "16678" uid: 264694fb-1046-447b-8a68-4059fbf461d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:10:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4aae5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: 4ab5a485-13aa-4d51-ae9e-bf8ed4f22b73 kind: Event lastTimestamp: "2026-07-10T19:11:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:10:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:41Z" name: success-200-isvc-4aae5-predictor.18c10364346ec146 namespace: kserve-ci-e2e-test resourceVersion: "16680" uid: 21d096bf-ab96-476f-9b06-75d56fc1f660 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: 7fac5ec4-6f46-4a1c-8cd2-d711960937ce kind: Event lastTimestamp: "2026-07-10T19:11:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:13Z" name: success-200-isvc-4aae5.18c1035d35cecc47 namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: a044cbdb-9664-40b5-a165-6c92c7d8d078 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: 7fac5ec4-6f46-4a1c-8cd2-d711960937ce kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: 'Failed to update status for InferenceService "success-200-isvc-4aae5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4aae5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-4aae5.18c1035d36345471 namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: 6fe53010-c6ae-440b-9a2b-be8c20ea74d0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: 7fac5ec4-6f46-4a1c-8cd2-d711960937ce kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4aae5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-4aae5.18c1035d36351bf5 namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: eb0c789b-c1c5-4d6f-afaf-6db06e2d8f9d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "16399" uid: 7fac5ec4-6f46-4a1c-8cd2-d711960937ce kind: Event lastTimestamp: "2026-07-10T19:11:13Z" message: InferenceService [success-200-isvc-4aae5] is Ready metadata: creationTimestamp: "2026-07-10T19:11:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:13Z" name: success-200-isvc-4aae5.18c103681e2a5ff5 namespace: kserve-ci-e2e-test resourceVersion: "16400" uid: bed0786e-ec68-4c46-b2ab-9634c7e3e17e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:30:16.497148Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: aadf3a41-21b5-4f85-8bad-65cce3ff530a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:30:16Z" name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc.18c1047253facf2b namespace: kserve-ci-e2e-test resourceVersion: "25732" uid: 706b6d76-f501-4b45-859a-b169193d3f86 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: aadf3a41-21b5-4f85-8bad-65cce3ff530a kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:30:16Z" name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc.18c104726dae6e56 namespace: kserve-ci-e2e-test resourceVersion: "25781" uid: 184babcc-9ba3-488e-8caf-51b0cb3bb7b8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: aadf3a41-21b5-4f85-8bad-65cce3ff530a kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:16Z" name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc.18c104726f20b5fa namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: 8bbcb088-70cb-4092-8006-6e81f363e938 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: aadf3a41-21b5-4f85-8bad-65cce3ff530a kind: Event lastTimestamp: "2026-07-10T19:30:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:17Z" name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc.18c10472722e2e65 namespace: kserve-ci-e2e-test resourceVersion: "25786" uid: e1df80b7-af63-4ed4-bbc6-dfd14c441c9a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: aadf3a41-21b5-4f85-8bad-65cce3ff530a kind: Event lastTimestamp: "2026-07-10T19:30:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:17Z" name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc.18c1047272a3ed84 namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: b74491b9-2e47-468c-8699-8e78de19f802 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: aadf3a41-21b5-4f85-8bad-65cce3ff530a kind: Event lastTimestamp: "2026-07-10T19:30:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:17Z" name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc.18c1047272ad9f2a namespace: kserve-ci-e2e-test resourceVersion: "25788" uid: c26874c8-6be1-4cad-8ce9-c1d72f22aa19 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: aadf3a41-21b5-4f85-8bad-65cce3ff530a kind: Event lastTimestamp: "2026-07-10T19:30:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:17Z" name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc.18c10472756e62f9 namespace: kserve-ci-e2e-test resourceVersion: "25789" uid: 4a15e383-a23f-4185-816d-6aaab7372952 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: aadf3a41-21b5-4f85-8bad-65cce3ff530a kind: Event lastTimestamp: "2026-07-10T19:30:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:17Z" name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc.18c1047275dd504d namespace: kserve-ci-e2e-test resourceVersion: "25790" uid: 1117d054-2abe-4cf4-bd9e-68a35516a510 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: aadf3a41-21b5-4f85-8bad-65cce3ff530a kind: Event lastTimestamp: "2026-07-10T19:30:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:54Z" name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc.18c10472d5a0641d namespace: kserve-ci-e2e-test resourceVersion: "26263" uid: 79e40b91-e1cd-4ade-99a8-ca15328ff648 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: aadf3a41-21b5-4f85-8bad-65cce3ff530a kind: Event lastTimestamp: "2026-07-10T19:31:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:26Z" name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc.18c10482afe07351 namespace: kserve-ci-e2e-test resourceVersion: "26610" uid: a61c0c11-9005-450d-a2b5-3616aa94eb0e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: aadf3a41-21b5-4f85-8bad-65cce3ff530a kind: Event lastTimestamp: "2026-07-10T19:31:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:26Z" name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc.18c10482afe10245 namespace: kserve-ci-e2e-test resourceVersion: "26612" uid: f9ff0a8f-d171-4793-8252-9852fa791be9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: aadf3a41-21b5-4f85-8bad-65cce3ff530a kind: Event lastTimestamp: "2026-07-10T19:31:29Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:31:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:29Z" name: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc.18c104835de38246 namespace: kserve-ci-e2e-test resourceVersion: "26746" uid: 60027595-434f-4c88-8e28-d319fd021114 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4b29b-predictor-68c4fbfdb9 namespace: kserve-ci-e2e-test resourceVersion: "25704" uid: 5a1ef5ba-bb44-4e6a-881f-6f5ca3c97645 kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: 'Created pod: success-200-isvc-4b29b-predictor-68c4fbfdb9-txgpc' metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:30:16Z" name: success-200-isvc-4b29b-predictor-68c4fbfdb9.18c1047253659602 namespace: kserve-ci-e2e-test resourceVersion: "25722" uid: ac218500-3aff-434e-a2de-770f306d95ca reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4b29b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25698" uid: 48b39721-a0e0-49c7-a2ef-2b437a99633a kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: Scaled up replica set success-200-isvc-4b29b-predictor-68c4fbfdb9 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:30:16Z" name: success-200-isvc-4b29b-predictor.18c104724fbdecd8 namespace: kserve-ci-e2e-test resourceVersion: "25707" uid: ebb082dc-12e6-45d0-a83c-e9bb28b818ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4b29b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25700" uid: c1165d25-3d45-41f2-8029-6c28a2b36a2b kind: Event lastTimestamp: "2026-07-10T19:30:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:30:46Z" name: success-200-isvc-4b29b-predictor.18c10475ce20431d namespace: kserve-ci-e2e-test resourceVersion: "26192" uid: a2e640c0-1ccd-4b76-aa3f-56e54cc32f2d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4b29b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25700" uid: c1165d25-3d45-41f2-8029-6c28a2b36a2b kind: Event lastTimestamp: "2026-07-10T19:30:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:30:46Z" name: success-200-isvc-4b29b-predictor.18c10475ce8b9449 namespace: kserve-ci-e2e-test resourceVersion: "26193" uid: 7cb03596-5d31-48a6-88fe-6ab14277f632 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:31:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4b29b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: c1165d25-3d45-41f2-8029-6c28a2b36a2b kind: Event lastTimestamp: "2026-07-10T19:31:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:16Z" name: success-200-isvc-4b29b-predictor.18c1047ccbea3e4b namespace: kserve-ci-e2e-test resourceVersion: "26456" uid: 3b14ee09-1daa-4c24-a1d3-f02bc2d37eb1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:31:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4b29b-predictor namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: c1165d25-3d45-41f2-8029-6c28a2b36a2b kind: Event lastTimestamp: "2026-07-10T19:31:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:16Z" name: success-200-isvc-4b29b-predictor.18c1047ccc4a6508 namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 0478ae77-2f64-409b-a79c-bdbdfd6387e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4b29b namespace: kserve-ci-e2e-test resourceVersion: "25684" uid: 0fba7515-8187-4e4f-98ef-2e085fe6b43d kind: Event lastTimestamp: "2026-07-10T19:31:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:04Z" name: success-200-isvc-4b29b.18c104724fc33827 namespace: kserve-ci-e2e-test resourceVersion: "26359" uid: 5553c934-92a7-4c4c-bbde-91b2c4402858 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4b29b namespace: kserve-ci-e2e-test resourceVersion: "25684" uid: 0fba7515-8187-4e4f-98ef-2e085fe6b43d kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-4b29b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4b29b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:30:16Z" name: success-200-isvc-4b29b.18c104725059678c namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 164313e9-d343-491c-931d-f9d9902d4032 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4b29b namespace: kserve-ci-e2e-test resourceVersion: "25684" uid: 0fba7515-8187-4e4f-98ef-2e085fe6b43d kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4b29b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:30:16Z" name: success-200-isvc-4b29b.18c104725059bc08 namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 866af434-7c1c-44f8-8ffd-ce451dede76c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4b29b namespace: kserve-ci-e2e-test resourceVersion: "26357" uid: 0fba7515-8187-4e4f-98ef-2e085fe6b43d kind: Event lastTimestamp: "2026-07-10T19:31:04Z" message: InferenceService [success-200-isvc-4b29b] is Ready metadata: creationTimestamp: "2026-07-10T19:31:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:04Z" name: success-200-isvc-4b29b.18c1047d90fe9d3e namespace: kserve-ci-e2e-test resourceVersion: "26358" uid: 9c210527-86ea-4a2f-a83b-c0aa6858d992 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:21:27.689392Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: 7565991f-b712-459f-8763-d4f09111de7f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5e7fe-predictor-68b56c976c-rx55x to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:21:27Z" name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x.18c103f73494db2e namespace: kserve-ci-e2e-test resourceVersion: "21511" uid: f966be52-3678-4239-9886-d3b39706466a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 7565991f-b712-459f-8763-d4f09111de7f kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-5e7fe-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:27Z" name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x.18c103f741c72ca3 namespace: kserve-ci-e2e-test resourceVersion: "21547" uid: 18d7a84b-f75e-4c1e-884d-b35dcb3dc316 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: 7565991f-b712-459f-8763-d4f09111de7f kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:21:28Z" name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x.18c103f7729a1f2b namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: 684c10a9-09ff-4e7f-90ef-3e0c6344eb45 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 7565991f-b712-459f-8763-d4f09111de7f kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:28Z" name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x.18c103f77427a07b namespace: kserve-ci-e2e-test resourceVersion: "21564" uid: 0d5e200c-c004-40d0-8a0e-cab864af2aad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 7565991f-b712-459f-8763-d4f09111de7f kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:28Z" name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x.18c103f777f58d51 namespace: kserve-ci-e2e-test resourceVersion: "21566" uid: 9d34569a-234f-4987-95b6-57dda5926cde reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 7565991f-b712-459f-8763-d4f09111de7f kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:28Z" name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x.18c103f7788d62e3 namespace: kserve-ci-e2e-test resourceVersion: "21567" uid: 87be43cf-6987-48ce-b7f4-e6ce20c60b40 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 7565991f-b712-459f-8763-d4f09111de7f kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:28Z" name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x.18c103f778970826 namespace: kserve-ci-e2e-test resourceVersion: "21569" uid: 081215f6-67b6-42d9-92df-cb47e06b7ffd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 7565991f-b712-459f-8763-d4f09111de7f kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:28Z" name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x.18c103f77ba36238 namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: 34347f0e-a114-44fe-a4ec-f8c11c649534 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 7565991f-b712-459f-8763-d4f09111de7f kind: Event lastTimestamp: "2026-07-10T19:21:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:28Z" name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x.18c103f77c28c560 namespace: kserve-ci-e2e-test resourceVersion: "21575" uid: f2e4d783-1d95-4e5f-8dc5-986f9318a092 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 7565991f-b712-459f-8763-d4f09111de7f kind: Event lastTimestamp: "2026-07-10T19:22:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:06Z" name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x.18c103f7e4f9df73 namespace: kserve-ci-e2e-test resourceVersion: "21943" uid: eefd4c82-8a90-4b53-8280-c161ce54417f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 7565991f-b712-459f-8763-d4f09111de7f kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:42Z" name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x.18c1047860a09f56 namespace: kserve-ci-e2e-test resourceVersion: "26052" uid: 9cffef49-933d-4ed5-8525-70c22134a818 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 7565991f-b712-459f-8763-d4f09111de7f kind: Event lastTimestamp: "2026-07-10T19:30:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:42Z" name: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x.18c1047860a13333 namespace: kserve-ci-e2e-test resourceVersion: "26053" uid: 6f98a67a-daf7-4022-8180-880a8869dc11 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5e7fe-predictor-68b56c976c namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: a60dd711-0e56-4ea5-8697-b0a0158087bd kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: 'Created pod: success-200-isvc-5e7fe-predictor-68b56c976c-rx55x' metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:27Z" name: success-200-isvc-5e7fe-predictor-68b56c976c.18c103f7342ef726 namespace: kserve-ci-e2e-test resourceVersion: "21509" uid: 63566332-d183-4e77-b089-af52eeafc4b9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5e7fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: 4754c67e-326e-47ad-bdae-01eaa0112a62 kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: Scaled up replica set success-200-isvc-5e7fe-predictor-68b56c976c from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:27Z" name: success-200-isvc-5e7fe-predictor.18c103f7322be7d9 namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: 4356429c-0c07-4197-9e7b-4d125ab95749 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5e7fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: a9d78e8f-736c-4b69-8908-53b8246489ae kind: Event lastTimestamp: "2026-07-10T19:21:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:57Z" name: success-200-isvc-5e7fe-predictor.18c103fab109afe9 namespace: kserve-ci-e2e-test resourceVersion: "21841" uid: 2245e84b-1fba-496d-8adf-87241eb84232 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5e7fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: a9d78e8f-736c-4b69-8908-53b8246489ae kind: Event lastTimestamp: "2026-07-10T19:21:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:57Z" name: success-200-isvc-5e7fe-predictor.18c103fab16df17f namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: b5eba61b-9a4e-4a5a-bfef-7ae1e5cb3211 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:22:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5e7fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21711" uid: a9d78e8f-736c-4b69-8908-53b8246489ae kind: Event lastTimestamp: "2026-07-10T19:22:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:22:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:27Z" name: success-200-isvc-5e7fe-predictor.18c10401af1c2957 namespace: kserve-ci-e2e-test resourceVersion: "22128" uid: 5f98cc31-5a5e-4067-a1db-c4c8f9ece3a2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:22:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5e7fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21711" uid: a9d78e8f-736c-4b69-8908-53b8246489ae kind: Event lastTimestamp: "2026-07-10T19:22:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:22:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:27Z" name: success-200-isvc-5e7fe-predictor.18c10401af842855 namespace: kserve-ci-e2e-test resourceVersion: "22132" uid: 0718a277-3981-4a53-a4b5-8abf29fc856f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5e7fe namespace: kserve-ci-e2e-test resourceVersion: "21457" uid: a4d1ba8c-b838-4955-8296-5d2940dff194 kind: Event lastTimestamp: "2026-07-10T19:22:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:22:16Z" name: success-200-isvc-5e7fe.18c103f73230cc7a namespace: kserve-ci-e2e-test resourceVersion: "22045" uid: 1b12e5c6-f50d-4818-b406-ab32e1ae37fe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5e7fe namespace: kserve-ci-e2e-test resourceVersion: "21457" uid: a4d1ba8c-b838-4955-8296-5d2940dff194 kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: 'Failed to update status for InferenceService "success-200-isvc-5e7fe": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5e7fe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:27Z" name: success-200-isvc-5e7fe.18c103f732f4cbdd namespace: kserve-ci-e2e-test resourceVersion: "21498" uid: d3a193c7-c70c-4933-9989-b7506eea5963 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5e7fe namespace: kserve-ci-e2e-test resourceVersion: "21457" uid: a4d1ba8c-b838-4955-8296-5d2940dff194 kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5e7fe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:27Z" name: success-200-isvc-5e7fe.18c103f732f52c5b namespace: kserve-ci-e2e-test resourceVersion: "21501" uid: f7f80993-9f2a-4cef-8215-59d0c616ed29 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5e7fe namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: a4d1ba8c-b838-4955-8296-5d2940dff194 kind: Event lastTimestamp: "2026-07-10T19:22:16Z" message: InferenceService [success-200-isvc-5e7fe] is Ready metadata: creationTimestamp: "2026-07-10T19:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:22:16Z" name: success-200-isvc-5e7fe.18c10402a1ca57cb namespace: kserve-ci-e2e-test resourceVersion: "22044" uid: 4cb4f244-5a05-419f-bb04-19fa9e385dbb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:08:52.219448Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:08:52Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c103474f106e61 namespace: kserve-ci-e2e-test resourceVersion: "14622" uid: 280039e2-1b62-4344-84ea-3f6db2277033 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ac00e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:08:52Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c103475aceefbb namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: be10b951-f9ff-4b74-b0ae-5d6016e857ad reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:08:54Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:08:54Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c10347c6525a57 namespace: kserve-ci-e2e-test resourceVersion: "14769" uid: 14a354b0-6783-4c4c-806e-ae61e7111d4c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:08:54Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1722" metadata: creationTimestamp: "2026-07-10T19:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:08:54Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c10347c8fd52f1 namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: 65a4d598-4428-4d74-8c93-abea6c86ba02 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:09:10Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1722" in 16.13s (16.13s including waiting). Image size: 1335614560 bytes.' metadata: creationTimestamp: "2026-07-10T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:10Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c1034b8a7596de namespace: kserve-ci-e2e-test resourceVersion: "14900" uid: a8d5f7bf-eb92-4b4a-a1ab-5fe6cef43198 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:09:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:10Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c1034b8e57482d namespace: kserve-ci-e2e-test resourceVersion: "14905" uid: 3594d70a-fabd-47c4-b2cf-ccfa5b908af2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:09:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:10Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c1034b8ed2e725 namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 609155de-2c75-4397-b106-49f7742cfade reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:09:10Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-10T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:10Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c1034b8ee1c7c0 namespace: kserve-ci-e2e-test resourceVersion: "14908" uid: a82e1e4d-1cf7-4817-a639-c371817f71cf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:09:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.603s (2.603s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-10T19:09:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:13Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c1034c2a09d58d namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: 1f819f4d-b7a3-4096-bb56-77c7e2691d5b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:09:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:09:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:13Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c1034c2e46f1c8 namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: 0bbbef66-dc0f-4e86-8c8f-b41eb8ae39f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:09:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:09:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:13Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c1034c2ec614a3 namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: 4da257b0-bdce-46ff-91dd-66aff3ae0c74 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-10T19:09:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:10:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:09:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:29Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c1034c51fef4e5 namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: 26c5735e-b1f5-41dc-b2a0-ec1a00e5e5f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c1035d32f1cb83 namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: 429212fc-dad9-4563-b884-ceac5de45607 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:26Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c1035d32f25322 namespace: kserve-ci-e2e-test resourceVersion: "15673" uid: d0c81272-0061-45d0-a506-228a4a5f0c71 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 5a961d5d-c8a2-483f-82c4-d63f496e434f kind: Event lastTimestamp: "2026-07-10T19:10:29Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:10:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:29Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg.18c1035e044477db namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: 82be38a7-724b-46d0-9223-5189be0772cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ac00e-predictor-5b8ff4ff6f namespace: kserve-ci-e2e-test resourceVersion: "14611" uid: e2b9b8a4-1acb-4180-8ff7-159258f1807e kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: 'Created pod: success-200-isvc-ac00e-predictor-5b8ff4ff6f-qrgbg' metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:08:52Z" name: success-200-isvc-ac00e-predictor-5b8ff4ff6f.18c103474e895914 namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: b9c52b47-a91e-499c-9897-6db1cad4c29f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ac00e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14607" uid: 8c1023a2-b9b0-43e9-bd20-f83f0a292c0e kind: Event lastTimestamp: "2026-07-10T19:08:52Z" message: Scaled up replica set success-200-isvc-ac00e-predictor-5b8ff4ff6f from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:08:52Z" name: success-200-isvc-ac00e-predictor.18c103474c5dcc9a namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: 186332a0-7df7-48f2-b2a4-0e5d9b85a517 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:09:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ac00e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14608" uid: fd792723-205f-474d-acdf-4e7cc9a43f16 kind: Event lastTimestamp: "2026-07-10T19:09:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:09:52Z" name: success-200-isvc-ac00e-predictor.18c1034acacf963e namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: 9557af01-38df-4d9d-800b-80f8ccc65088 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:09:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ac00e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14608" uid: fd792723-205f-474d-acdf-4e7cc9a43f16 kind: Event lastTimestamp: "2026-07-10T19:09:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:09:52Z" name: success-200-isvc-ac00e-predictor.18c1034acb2f0850 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 2210d6da-5881-4d25-89ac-6ded7d64dd4f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:09:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ac00e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: fd792723-205f-474d-acdf-4e7cc9a43f16 kind: Event lastTimestamp: "2026-07-10T19:10:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:09:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:22Z" name: success-200-isvc-ac00e-predictor.18c10351c8901883 namespace: kserve-ci-e2e-test resourceVersion: "15606" uid: 92745ac4-bfd6-4d67-8998-f51b81c4b6de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:09:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ac00e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: fd792723-205f-474d-acdf-4e7cc9a43f16 kind: Event lastTimestamp: "2026-07-10T19:10:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:09:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:22Z" name: success-200-isvc-ac00e-predictor.18c10351c8f88050 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 6a4ecdf1-a7b1-4640-8d16-11a66a381e12 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:08:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ac00e namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: 94a8b9c1-2f51-4e61-be8b-32c6312a5e0c kind: Event lastTimestamp: "2026-07-10T19:09:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:09:59Z" name: success-200-isvc-ac00e.18c103474c60faa5 namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 4c0f1ba5-2782-4e8d-abcd-4c1b73e67b0a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ac00e namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: 94a8b9c1-2f51-4e61-be8b-32c6312a5e0c kind: Event lastTimestamp: "2026-07-10T19:09:59Z" message: InferenceService [success-200-isvc-ac00e] is Ready metadata: creationTimestamp: "2026-07-10T19:09:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:09:59Z" name: success-200-isvc-ac00e.18c1035710ce82a8 namespace: kserve-ci-e2e-test resourceVersion: "15376" uid: 9338cf47-4203-4ea7-9e80-08019c38bbad reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:20:51.586260Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 08e9d3ba-f634-4ed4-b81e-d82a66f50c3c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:20:51Z" name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c.18c103eeccaabdcb namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: e17fe774-10e6-46a5-a0db-4f47f49b5de1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: 08e9d3ba-f634-4ed4-b81e-d82a66f50c3c kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:20:52Z" name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c.18c103eee6a58308 namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: aa44427c-1c42-4a23-9495-ee9aff7ecf56 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 08e9d3ba-f634-4ed4-b81e-d82a66f50c3c kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c.18c103eee80e488f namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: 2879c0a5-5c76-42da-9f95-d943bd8fbc97 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 08e9d3ba-f634-4ed4-b81e-d82a66f50c3c kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c.18c103eeebc35223 namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: 0d2bfa98-1a12-4ee8-b4a7-1ff4b7d68d37 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 08e9d3ba-f634-4ed4-b81e-d82a66f50c3c kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c.18c103eeecd09b9e namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 53d413e1-c2db-406c-bab7-c95caea75573 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 08e9d3ba-f634-4ed4-b81e-d82a66f50c3c kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c.18c103eeecdc5ef4 namespace: kserve-ci-e2e-test resourceVersion: "21060" uid: 7203844d-d207-43bb-ac76-d9b587f89fbb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 08e9d3ba-f634-4ed4-b81e-d82a66f50c3c kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c.18c103eef07044ec namespace: kserve-ci-e2e-test resourceVersion: "21062" uid: 536145f8-cb02-4400-b91a-40cc2ce63399 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 08e9d3ba-f634-4ed4-b81e-d82a66f50c3c kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c.18c103eef0e45447 namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: 250f5120-0cf3-4d84-a1bd-7ebbac78957e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 08e9d3ba-f634-4ed4-b81e-d82a66f50c3c kind: Event lastTimestamp: "2026-07-10T19:30:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:18Z" name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c.18c103ef0189e40f namespace: kserve-ci-e2e-test resourceVersion: "25813" uid: c72858e5-6a56-4143-8807-d3fc75096b67 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 08e9d3ba-f634-4ed4-b81e-d82a66f50c3c kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:16Z" name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c.18c10472507c5646 namespace: kserve-ci-e2e-test resourceVersion: "25712" uid: b80758d8-1d5c-47cb-8217-02fc63d6aec2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 08e9d3ba-f634-4ed4-b81e-d82a66f50c3c kind: Event lastTimestamp: "2026-07-10T19:30:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:16Z" name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c.18c10472507d0ba3 namespace: kserve-ci-e2e-test resourceVersion: "25716" uid: a1a6dc27-a0a7-41f7-9986-8f4bd6d83eb1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 08e9d3ba-f634-4ed4-b81e-d82a66f50c3c kind: Event lastTimestamp: "2026-07-10T19:30:18Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:30:18Z" name: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c.18c10472ca5918de namespace: kserve-ci-e2e-test resourceVersion: "25812" uid: 07fecc8f-f089-4b34-9d6f-80a97e965271 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d232d-predictor-5459ccd6b9 namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: 71d10019-3284-40d6-9cbb-a0127b051b74 kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: 'Created pod: success-200-isvc-d232d-predictor-5459ccd6b9-ldx4c' metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:51Z" name: success-200-isvc-d232d-predictor-5459ccd6b9.18c103eecc37cfbd namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: 4f2807e7-0db8-48ba-b6c3-77c74ca9012d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d232d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20998" uid: 2a502ecd-53ba-42b7-b4cf-5283185b2ee0 kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: Scaled up replica set success-200-isvc-d232d-predictor-5459ccd6b9 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:51Z" name: success-200-isvc-d232d-predictor.18c103eeca1d0b7f namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: 9b708154-aaf6-4260-bcb6-46f537dbc575 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d232d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: 1890eb7d-9750-4376-b8d1-50672115375a kind: Event lastTimestamp: "2026-07-10T19:21:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:21Z" name: success-200-isvc-d232d-predictor.18c103f2489dff00 namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: 28df350a-9a31-4274-9bd6-3948792f5ae1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d232d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: 1890eb7d-9750-4376-b8d1-50672115375a kind: Event lastTimestamp: "2026-07-10T19:21:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:21Z" name: success-200-isvc-d232d-predictor.18c103f248fcdb05 namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: 6e32fbc2-1e41-4a0f-a9c9-5440309f0468 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d232d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: 1890eb7d-9750-4376-b8d1-50672115375a kind: Event lastTimestamp: "2026-07-10T19:22:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:21Z" name: success-200-isvc-d232d-predictor.18c103f94668c9d7 namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: 6f2fe448-795a-427b-98a5-1a5dcd5d370a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d232d-predictor namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: 1890eb7d-9750-4376-b8d1-50672115375a kind: Event lastTimestamp: "2026-07-10T19:22:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:21Z" name: success-200-isvc-d232d-predictor.18c103f946c8dac7 namespace: kserve-ci-e2e-test resourceVersion: "22084" uid: 516e83bf-ebed-45f1-bc42-58e2836db141 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d232d namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: e86142c0-459f-4ded-bdde-16c0e70f0a85 kind: Event lastTimestamp: "2026-07-10T19:21:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:48Z" name: success-200-isvc-d232d.18c103eeca2865d2 namespace: kserve-ci-e2e-test resourceVersion: "21765" uid: 99abb343-ae29-4000-8bc9-80e66fd6efd0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d232d namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: e86142c0-459f-4ded-bdde-16c0e70f0a85 kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-d232d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d232d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:51Z" name: success-200-isvc-d232d.18c103eecaa8f458 namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: 3c7f0ba6-bdc0-4753-8b4f-269338c8a996 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d232d namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: e86142c0-459f-4ded-bdde-16c0e70f0a85 kind: Event lastTimestamp: "2026-07-10T19:20:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d232d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:51Z" name: success-200-isvc-d232d.18c103eecaa93975 namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: c623a430-b68d-4ff5-a80a-e88e13a42233 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d232d namespace: kserve-ci-e2e-test resourceVersion: "21763" uid: e86142c0-459f-4ded-bdde-16c0e70f0a85 kind: Event lastTimestamp: "2026-07-10T19:21:48Z" message: InferenceService [success-200-isvc-d232d] is Ready metadata: creationTimestamp: "2026-07-10T19:21:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:48Z" name: success-200-isvc-d232d.18c103fc10912574 namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 9cc9b00c-c1b7-4fa6-9a4f-0085fe4ed676 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:20:17.352744Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 17b39e35-525f-4039-88b4-50169241e202 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d30ee-predictor-585b9dc857-dqrvt to ip-10-0-139-20.ec2.internal metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:20:17Z" name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt.18c103e6d430da8a namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 4b557c48-65d4-4fdc-bc38-65222ff5189a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 17b39e35-525f-4039-88b4-50169241e202 kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d30ee-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:17Z" name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt.18c103e6df93cf9b namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: d2890bac-3851-454f-8960-117b7d4481f9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 17b39e35-525f-4039-88b4-50169241e202 kind: Event lastTimestamp: "2026-07-10T19:20:18Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:20:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:20:18Z" name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt.18c103e711c48441 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: 518d150c-abd0-4824-b444-03ac02a34bcc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 17b39e35-525f-4039-88b4-50169241e202 kind: Event lastTimestamp: "2026-07-10T19:20:18Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:20:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:18Z" name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt.18c103e7133d15ea namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: bb3309a2-974e-4f94-baf8-8eb79ebde3f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 17b39e35-525f-4039-88b4-50169241e202 kind: Event lastTimestamp: "2026-07-10T19:20:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:20:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:18Z" name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt.18c103e71663c8e8 namespace: kserve-ci-e2e-test resourceVersion: "20596" uid: 1813dbec-d622-47de-9faf-f6400691f9bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 17b39e35-525f-4039-88b4-50169241e202 kind: Event lastTimestamp: "2026-07-10T19:20:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:20:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:18Z" name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt.18c103e716d66b4b namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 33dba202-2e68-42f7-9a56-e9421a3b6b99 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 17b39e35-525f-4039-88b4-50169241e202 kind: Event lastTimestamp: "2026-07-10T19:20:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:20:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:18Z" name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt.18c103e716e00c26 namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 9910d280-2e3f-46e0-8bcf-d6d40ace3f1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 17b39e35-525f-4039-88b4-50169241e202 kind: Event lastTimestamp: "2026-07-10T19:20:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:20:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:18Z" name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt.18c103e71a214155 namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: ec07ab25-904d-43e4-9d6d-ff7ad34029f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 17b39e35-525f-4039-88b4-50169241e202 kind: Event lastTimestamp: "2026-07-10T19:20:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:20:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:18Z" name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt.18c103e71a970c7e namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: b2f09d32-8b1b-40be-be2f-b6cde6b2931e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 17b39e35-525f-4039-88b4-50169241e202 kind: Event lastTimestamp: "2026-07-10T19:20:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:56Z" name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt.18c103e785843625 namespace: kserve-ci-e2e-test resourceVersion: "21121" uid: 7423c484-b44f-4a61-8d91-ed17f66dcb06 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 17b39e35-525f-4039-88b4-50169241e202 kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:27Z" name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt.18c103f7338e4f8f namespace: kserve-ci-e2e-test resourceVersion: "21503" uid: 7b40ec47-9f61-4570-a2af-1fb96984ff14 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 17b39e35-525f-4039-88b4-50169241e202 kind: Event lastTimestamp: "2026-07-10T19:21:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:27Z" name: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt.18c103f7338f196a namespace: kserve-ci-e2e-test resourceVersion: "21505" uid: 5fba37f2-4512-4840-a798-8eee83ad96e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-20.ec2.internal source: component: kubelet host: ip-10-0-139-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d30ee-predictor-585b9dc857 namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 826a3a7b-a685-4588-9931-275755149a9b kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: 'Created pod: success-200-isvc-d30ee-predictor-585b9dc857-dqrvt' metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:17Z" name: success-200-isvc-d30ee-predictor-585b9dc857.18c103e6d35ca167 namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: 819b9773-da89-4290-b612-ddd7d8aba79c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d30ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "20501" uid: 3f79de5f-63f8-447d-8c58-5a70ee66fe7f kind: Event lastTimestamp: "2026-07-10T19:20:17Z" message: Scaled up replica set success-200-isvc-d30ee-predictor-585b9dc857 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:17Z" name: success-200-isvc-d30ee-predictor.18c103e6d09ddf84 namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: 6cb53f57-33ed-4665-94eb-3fac96e4a3a7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:20:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d30ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "20505" uid: ad47ee80-6081-4166-81c6-012ae951856f kind: Event lastTimestamp: "2026-07-10T19:20:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:20:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:47Z" name: success-200-isvc-d30ee-predictor.18c103ea4f09e1b1 namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: 59e03257-cefc-416a-96f5-d9e95a212ab5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:20:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d30ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "20505" uid: ad47ee80-6081-4166-81c6-012ae951856f kind: Event lastTimestamp: "2026-07-10T19:20:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:20:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:47Z" name: success-200-isvc-d30ee-predictor.18c103ea4f6b5233 namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 46450aa5-7ef6-47b9-87ee-9feab0e9d316 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d30ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "20743" uid: ad47ee80-6081-4166-81c6-012ae951856f kind: Event lastTimestamp: "2026-07-10T19:21:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:17Z" name: success-200-isvc-d30ee-predictor.18c103f14cf07ab5 namespace: kserve-ci-e2e-test resourceVersion: "21322" uid: d74cc6c0-7da6-453c-9ca1-b2df52c0676f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d30ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "20743" uid: ad47ee80-6081-4166-81c6-012ae951856f kind: Event lastTimestamp: "2026-07-10T19:21:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:17Z" name: success-200-isvc-d30ee-predictor.18c103f14d5f5d42 namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 2662e693-d1f9-42b8-90b7-59d77da8927d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:20:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d30ee namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: ec70a784-b92b-4ef3-906f-1594bb11259d kind: Event lastTimestamp: "2026-07-10T19:21:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:20:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:06Z" name: success-200-isvc-d30ee.18c103e6d0a4399f namespace: kserve-ci-e2e-test resourceVersion: "21224" uid: dcfda529-acee-4106-be82-644659629629 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d30ee namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: ec70a784-b92b-4ef3-906f-1594bb11259d kind: Event lastTimestamp: "2026-07-10T19:21:06Z" message: InferenceService [success-200-isvc-d30ee] is Ready metadata: creationTimestamp: "2026-07-10T19:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:06Z" name: success-200-isvc-d30ee.18c103f240b5d80b namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: b003b183-f8fe-45e1-a4a3-8f92a5d85e40 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:31:42.614017Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1c787-7d48d4c4f9-bzp49 namespace: kserve-ci-e2e-test resourceVersion: "26877" uid: 3c791eec-7293-4aba-83b9-5391cc1ecb15 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-1c787-7d48d4c4f9-bzp49 to ip-10-0-131-123.ec2.internal metadata: creationTimestamp: "2026-07-10T19:31:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:31:42Z" name: switch-graph-1c787-7d48d4c4f9-bzp49.18c1048660f1f30a namespace: kserve-ci-e2e-test resourceVersion: "26885" uid: e7704993-d26c-433c-8c93-9833b6208ed2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:43Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1c787-7d48d4c4f9-bzp49 namespace: kserve-ci-e2e-test resourceVersion: "26889" uid: 3c791eec-7293-4aba-83b9-5391cc1ecb15 kind: Event lastTimestamp: "2026-07-10T19:31:43Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:31:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:31:43Z" name: switch-graph-1c787-7d48d4c4f9-bzp49.18c104867a06d5ea namespace: kserve-ci-e2e-test resourceVersion: "26901" uid: ba2cf04e-6ba7-410b-b3f6-5ba3c536dc40 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1c787} kind: Pod name: switch-graph-1c787-7d48d4c4f9-bzp49 namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: 3c791eec-7293-4aba-83b9-5391cc1ecb15 kind: Event lastTimestamp: "2026-07-10T19:31:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" already present on machine metadata: creationTimestamp: "2026-07-10T19:31:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:43Z" name: switch-graph-1c787-7d48d4c4f9-bzp49.18c104867b57b631 namespace: kserve-ci-e2e-test resourceVersion: "26903" uid: dfa0a918-31f6-4ec2-98aa-693b7f31debd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1c787} kind: Pod name: switch-graph-1c787-7d48d4c4f9-bzp49 namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: 3c791eec-7293-4aba-83b9-5391cc1ecb15 kind: Event lastTimestamp: "2026-07-10T19:31:43Z" message: 'Created container: switch-graph-1c787' metadata: creationTimestamp: "2026-07-10T19:31:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:43Z" name: switch-graph-1c787-7d48d4c4f9-bzp49.18c104867f2c7a91 namespace: kserve-ci-e2e-test resourceVersion: "26904" uid: 1a830b3e-c06b-4097-b444-cb23a03c6f8d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1c787} kind: Pod name: switch-graph-1c787-7d48d4c4f9-bzp49 namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: 3c791eec-7293-4aba-83b9-5391cc1ecb15 kind: Event lastTimestamp: "2026-07-10T19:31:43Z" message: Started container switch-graph-1c787 metadata: creationTimestamp: "2026-07-10T19:31:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:31:43Z" name: switch-graph-1c787-7d48d4c4f9-bzp49.18c104867fe26571 namespace: kserve-ci-e2e-test resourceVersion: "26905" uid: bfa88fa6-ab5d-4a63-b48d-4654c6c72875 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1c787} kind: Pod name: switch-graph-1c787-7d48d4c4f9-bzp49 namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: 3c791eec-7293-4aba-83b9-5391cc1ecb15 kind: Event lastTimestamp: "2026-07-10T19:48:01Z" message: Stopping container switch-graph-1c787 metadata: creationTimestamp: "2026-07-10T19:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:48:01Z" name: switch-graph-1c787-7d48d4c4f9-bzp49.18c1056a5eb713ed namespace: kserve-ci-e2e-test resourceVersion: "33811" uid: 13c1ad6a-2d50-4b03-a7f7-4810d2100cc5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-10T19:48:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1c787} kind: Pod name: switch-graph-1c787-7d48d4c4f9-bzp49 namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: 3c791eec-7293-4aba-83b9-5391cc1ecb15 kind: Event lastTimestamp: "2026-07-10T19:48:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:48:24Z" name: switch-graph-1c787-7d48d4c4f9-bzp49.18c1056af98b8e42 namespace: kserve-ci-e2e-test resourceVersion: "34047" uid: 2b7e9d7f-36db-4f5f-b4e8-5d5506d43e58 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-1c787-7d48d4c4f9 namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: ac2b98ed-c05b-4601-92c1-6dc1cbed10dc kind: Event lastTimestamp: "2026-07-10T19:31:42Z" message: 'Created pod: switch-graph-1c787-7d48d4c4f9-bzp49' metadata: creationTimestamp: "2026-07-10T19:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:42Z" name: switch-graph-1c787-7d48d4c4f9.18c1048660657d60 namespace: kserve-ci-e2e-test resourceVersion: "26881" uid: 7f769860-11f7-4fc3-a54d-aed29c640bbb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-1c787 namespace: kserve-ci-e2e-test resourceVersion: "26871" uid: ce3cac16-7ceb-4446-b6f7-2c9d90aae2a5 kind: Event lastTimestamp: "2026-07-10T19:31:42Z" message: Scaled up replica set switch-graph-1c787-7d48d4c4f9 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:31:42Z" name: switch-graph-1c787.18c104865fb91db9 namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 5def79b4-515b-4e06-b9b7-eda5b229fc11 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:31:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-1c787 namespace: kserve-ci-e2e-test resourceVersion: "26891" uid: afc0ea24-ea83-40fe-b2f8-11eecef7abd9 kind: Event lastTimestamp: "2026-07-10T19:31:42Z" message: InferenceGraph [switch-graph-1c787] is Ready metadata: creationTimestamp: "2026-07-10T19:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:31:42Z" name: switch-graph-1c787.18c104866412da04 namespace: kserve-ci-e2e-test resourceVersion: "26893" uid: 2de4d29f-967a-4ef4-aee5-b72fd5f5fe1e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:31:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1c787 namespace: kserve-ci-e2e-test resourceVersion: "26872" uid: 38ee9a27-921a-4929-ba12-42b6abb42fd4 kind: Event lastTimestamp: "2026-07-10T19:32:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:12Z" name: switch-graph-1c787.18c10489de2f1d70 namespace: kserve-ci-e2e-test resourceVersion: "27145" uid: b11112af-6e7c-4c8f-92e9-f850d0d841c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:31:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1c787 namespace: kserve-ci-e2e-test resourceVersion: "26872" uid: 38ee9a27-921a-4929-ba12-42b6abb42fd4 kind: Event lastTimestamp: "2026-07-10T19:32:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:32:12Z" name: switch-graph-1c787.18c10489de96305f namespace: kserve-ci-e2e-test resourceVersion: "27146" uid: f1d7a0b8-e349-4954-b59d-d3b9f08b3b41 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:11:26.402322Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4aae5-cdb5c996f-pfs62 namespace: kserve-ci-e2e-test resourceVersion: "16524" uid: 94da2fcc-2aa2-43fc-9a22-76af036e8afa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-4aae5-cdb5c996f-pfs62 to ip-10-0-131-123.ec2.internal metadata: creationTimestamp: "2026-07-10T19:11:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:11:26Z" name: switch-graph-4aae5-cdb5c996f-pfs62.18c1036b35143e5c namespace: kserve-ci-e2e-test resourceVersion: "16529" uid: daf4c2bc-32e3-44c1-87e0-4c263e074368 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:26Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4aae5-cdb5c996f-pfs62 namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 94da2fcc-2aa2-43fc-9a22-76af036e8afa kind: Event lastTimestamp: "2026-07-10T19:11:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-4aae5-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:26Z" name: switch-graph-4aae5-cdb5c996f-pfs62.18c1036b3fdd0109 namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: 8e0928fd-0565-4f38-98f1-0cbe0aba62a6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:27Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4aae5-cdb5c996f-pfs62 namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 94da2fcc-2aa2-43fc-9a22-76af036e8afa kind: Event lastTimestamp: "2026-07-10T19:11:27Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:11:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:11:27Z" name: switch-graph-4aae5-cdb5c996f-pfs62.18c1036b72ecc76d namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 8c869282-dfae-45d0-8f40-b76a796eb511 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4aae5} kind: Pod name: switch-graph-4aae5-cdb5c996f-pfs62 namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 94da2fcc-2aa2-43fc-9a22-76af036e8afa kind: Event lastTimestamp: "2026-07-10T19:11:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:27Z" name: switch-graph-4aae5-cdb5c996f-pfs62.18c1036b74325fcb namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: 259899f8-ad5e-449d-a9a0-de794aaf0895 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4aae5} kind: Pod name: switch-graph-4aae5-cdb5c996f-pfs62 namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 94da2fcc-2aa2-43fc-9a22-76af036e8afa kind: Event lastTimestamp: "2026-07-10T19:11:27Z" message: 'Created container: switch-graph-4aae5' metadata: creationTimestamp: "2026-07-10T19:11:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:27Z" name: switch-graph-4aae5-cdb5c996f-pfs62.18c1036b77154ec8 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 7118ad82-b00c-4108-9f2d-49bafb7b7191 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4aae5} kind: Pod name: switch-graph-4aae5-cdb5c996f-pfs62 namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 94da2fcc-2aa2-43fc-9a22-76af036e8afa kind: Event lastTimestamp: "2026-07-10T19:11:27Z" message: Started container switch-graph-4aae5 metadata: creationTimestamp: "2026-07-10T19:11:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:27Z" name: switch-graph-4aae5-cdb5c996f-pfs62.18c1036b77814251 namespace: kserve-ci-e2e-test resourceVersion: "16560" uid: 17829d44-a894-402d-820f-fd4242fb0098 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4aae5} kind: Pod name: switch-graph-4aae5-cdb5c996f-pfs62 namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 94da2fcc-2aa2-43fc-9a22-76af036e8afa kind: Event lastTimestamp: "2026-07-10T19:19:41Z" message: Stopping container switch-graph-4aae5 metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:41Z" name: switch-graph-4aae5-cdb5c996f-pfs62.18c103de5fc21b17 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: c9132b25-2d9a-41c8-8ad9-bef057ec9597 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:19:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4aae5} kind: Pod name: switch-graph-4aae5-cdb5c996f-pfs62 namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 94da2fcc-2aa2-43fc-9a22-76af036e8afa kind: Event lastTimestamp: "2026-07-10T19:20:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:19:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:09Z" name: switch-graph-4aae5-cdb5c996f-pfs62.18c103df27df5f57 namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: 9ff50563-01b6-44d8-bd62-e00686d1fc0c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-4aae5-cdb5c996f namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: 39f7cdde-7e66-4169-a6e3-eba90a395bb5 kind: Event lastTimestamp: "2026-07-10T19:11:26Z" message: 'Created pod: switch-graph-4aae5-cdb5c996f-pfs62' metadata: creationTimestamp: "2026-07-10T19:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:26Z" name: switch-graph-4aae5-cdb5c996f.18c1036b34b795dd namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 00429796-4a80-4570-9a0c-5a4130f5f58e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: 5d644250-a8c6-4801-ba3f-19228fe7ee1f kind: Event lastTimestamp: "2026-07-10T19:11:26Z" message: Scaled up replica set switch-graph-4aae5-cdb5c996f from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:26Z" name: switch-graph-4aae5.18c1036b33ea13d2 namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: 575c6cbc-b4c0-44dc-9f02-3b4e84339dbf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: 232fd7ae-19cf-4c91-a979-816d0dc36d62 kind: Event lastTimestamp: "2026-07-10T19:11:26Z" message: InferenceGraph [switch-graph-4aae5] is Ready metadata: creationTimestamp: "2026-07-10T19:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:26Z" name: switch-graph-4aae5.18c1036b37c842ff namespace: kserve-ci-e2e-test resourceVersion: "16540" uid: 652b5845-1404-4f34-ae97-5e9909076628 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: 232fd7ae-19cf-4c91-a979-816d0dc36d62 kind: Event lastTimestamp: "2026-07-10T19:11:26Z" message: 'Failed to update status for InferenceGraph "switch-graph-4aae5": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-4aae5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:26Z" name: switch-graph-4aae5.18c1036b3a693b06 namespace: kserve-ci-e2e-test resourceVersion: "16542" uid: 50b27f5b-f204-42f3-aa4e-923018944974 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: 232fd7ae-19cf-4c91-a979-816d0dc36d62 kind: Event lastTimestamp: "2026-07-10T19:11:26Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-4aae5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:26Z" name: switch-graph-4aae5.18c1036b3a69807b namespace: kserve-ci-e2e-test resourceVersion: "16543" uid: 0f026036-6fd6-40f8-87dc-63e0b220a62c reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:11:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: ed291fa9-c260-4b9d-a783-19e8cb18ebe0 kind: Event lastTimestamp: "2026-07-10T19:12:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:11Z" name: switch-graph-4aae5.18c1036eb26aa211 namespace: kserve-ci-e2e-test resourceVersion: "16963" uid: daaffd80-21cd-465d-8995-7231929a0351 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:11:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-4aae5 namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: ed291fa9-c260-4b9d-a783-19e8cb18ebe0 kind: Event lastTimestamp: "2026-07-10T19:12:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:11Z" name: switch-graph-4aae5.18c1036eb2a42233 namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: fa2227da-cc9b-46e9-a791-77c1fb378796 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:10:11.951725Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ac00e-8445bbc489-kn8km namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: 14c9005c-bfc8-44a3-b8d0-c7a045bcd708 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-ac00e-8445bbc489-kn8km to ip-10-0-131-123.ec2.internal metadata: creationTimestamp: "2026-07-10T19:10:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:10:11Z" name: switch-graph-ac00e-8445bbc489-kn8km.18c10359df7a3f88 namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: e342f8b5-ac13-4c42-a607-bb621be9d9cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-85774cf8cf-8rglt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:12Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ac00e-8445bbc489-kn8km namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 14c9005c-bfc8-44a3-b8d0-c7a045bcd708 kind: Event lastTimestamp: "2026-07-10T19:10:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-ac00e-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:10:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:12Z" name: switch-graph-ac00e-8445bbc489-kn8km.18c10359eb83ec9d namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: b26234ad-e784-4942-a867-651f5e7c24e5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:12Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ac00e-8445bbc489-kn8km namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: 14c9005c-bfc8-44a3-b8d0-c7a045bcd708 kind: Event lastTimestamp: "2026-07-10T19:10:12Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:10:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:10:12Z" name: switch-graph-ac00e-8445bbc489-kn8km.18c1035a1c8996e4 namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: 9e4a0908-7c64-4b07-9e72-3cea96554af8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ac00e} kind: Pod name: switch-graph-ac00e-8445bbc489-kn8km namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 14c9005c-bfc8-44a3-b8d0-c7a045bcd708 kind: Event lastTimestamp: "2026-07-10T19:10:12Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" metadata: creationTimestamp: "2026-07-10T19:10:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:12Z" name: switch-graph-ac00e-8445bbc489-kn8km.18c1035a1dd79821 namespace: kserve-ci-e2e-test resourceVersion: "15524" uid: a92b5286-b929-4273-8808-0319a3d921a3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ac00e} kind: Pod name: switch-graph-ac00e-8445bbc489-kn8km namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 14c9005c-bfc8-44a3-b8d0-c7a045bcd708 kind: Event lastTimestamp: "2026-07-10T19:10:15Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" in 2.217s (2.217s including waiting). Image size: 218072665 bytes.' metadata: creationTimestamp: "2026-07-10T19:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:15Z" name: switch-graph-ac00e-8445bbc489-kn8km.18c1035aa202772a namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 743b4076-c005-4674-a311-cbd47ab974ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ac00e} kind: Pod name: switch-graph-ac00e-8445bbc489-kn8km namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 14c9005c-bfc8-44a3-b8d0-c7a045bcd708 kind: Event lastTimestamp: "2026-07-10T19:10:15Z" message: 'Created container: switch-graph-ac00e' metadata: creationTimestamp: "2026-07-10T19:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:15Z" name: switch-graph-ac00e-8445bbc489-kn8km.18c1035aa4d8fe70 namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: d93dd4f2-13a0-4bcf-932b-36a83f92f628 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ac00e} kind: Pod name: switch-graph-ac00e-8445bbc489-kn8km namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 14c9005c-bfc8-44a3-b8d0-c7a045bcd708 kind: Event lastTimestamp: "2026-07-10T19:10:15Z" message: Started container switch-graph-ac00e metadata: creationTimestamp: "2026-07-10T19:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:15Z" name: switch-graph-ac00e-8445bbc489-kn8km.18c1035aa563e46a namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: fe938562-1c93-409d-9f69-9a8598e3cfe6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ac00e} kind: Pod name: switch-graph-ac00e-8445bbc489-kn8km namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 14c9005c-bfc8-44a3-b8d0-c7a045bcd708 kind: Event lastTimestamp: "2026-07-10T19:10:26Z" message: Stopping container switch-graph-ac00e metadata: creationTimestamp: "2026-07-10T19:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:26Z" name: switch-graph-ac00e-8445bbc489-kn8km.18c1035d2a9e1b63 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: 2b27fc5d-5aec-4bc8-9317-f1c21d259aad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ac00e} kind: Pod name: switch-graph-ac00e-8445bbc489-kn8km namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 14c9005c-bfc8-44a3-b8d0-c7a045bcd708 kind: Event lastTimestamp: "2026-07-10T19:10:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:52Z" name: switch-graph-ac00e-8445bbc489-kn8km.18c1035d6653ded3 namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 3369a974-cc01-4de2-a021-be722ddd7357 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-123.ec2.internal source: component: kubelet host: ip-10-0-131-123.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-ac00e-8445bbc489 namespace: kserve-ci-e2e-test resourceVersion: "15490" uid: ba8ccf95-584e-44be-a80d-b78fbca73065 kind: Event lastTimestamp: "2026-07-10T19:10:11Z" message: 'Created pod: switch-graph-ac00e-8445bbc489-kn8km' metadata: creationTimestamp: "2026-07-10T19:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:11Z" name: switch-graph-ac00e-8445bbc489.18c10359df051adc namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: e1075141-f74b-46ae-9e49-34361a1c1730 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-ac00e namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: 9d6046b1-ca4f-40d3-9db5-49422212c767 kind: Event lastTimestamp: "2026-07-10T19:10:11Z" message: Scaled up replica set switch-graph-ac00e-8445bbc489 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:11Z" name: switch-graph-ac00e.18c10359de201950 namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: 32334ed8-2026-4d11-b7ca-a124ed55488b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ac00e namespace: kserve-ci-e2e-test resourceVersion: "15505" uid: 21f2250c-2ae7-4eaf-9d02-e0266e68939b kind: Event lastTimestamp: "2026-07-10T19:10:12Z" message: InferenceGraph [switch-graph-ac00e] is Ready metadata: creationTimestamp: "2026-07-10T19:10:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:12Z" name: switch-graph-ac00e.18c10359e40c39bf namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: d192bf24-cc6a-4d38-9251-33c7eaa1f6a2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ac00e namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: 21f2250c-2ae7-4eaf-9d02-e0266e68939b kind: Event lastTimestamp: "2026-07-10T19:10:12Z" message: 'Failed to update status for InferenceGraph "switch-graph-ac00e": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-ac00e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:10:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:12Z" name: switch-graph-ac00e.18c10359e6402e2d namespace: kserve-ci-e2e-test resourceVersion: "15509" uid: c810f946-8bc0-4e06-b60d-446c01f9e78d reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ac00e namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: 21f2250c-2ae7-4eaf-9d02-e0266e68939b kind: Event lastTimestamp: "2026-07-10T19:10:12Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-ac00e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:10:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:12Z" name: switch-graph-ac00e.18c10359e640b756 namespace: kserve-ci-e2e-test resourceVersion: "15510" uid: 2cd6bbf0-d6ed-4c0e-bffc-a2f105520b7d reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning kind: EventList metadata: resourceVersion: "34072"