--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:38:53.654263Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-6304f-66c85585fd-jtg67 namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: d36fe325-e6e4-464a-876a-5eefa53603ef kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-6304f-66c85585fd-jtg67 to ip-10-0-128-224.ec2.internal metadata: creationTimestamp: "2026-09-18T16:38:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:38:53Z" name: ensemble-graph-6304f-66c85585fd-jtg67.18d677b7e6625181 namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: 938d515b-1c67-4e03-89d0-6ad583d9d00a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:53Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-6304f-66c85585fd-jtg67 namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: d36fe325-e6e4-464a-876a-5eefa53603ef kind: Event lastTimestamp: "2026-09-18T16:38:53Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-6304f-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:38:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:53Z" name: ensemble-graph-6304f-66c85585fd-jtg67.18d677b7eff5bfbe namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: 3a14b28a-6aa8-431c-b01e-1df17ab4b450 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:54Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-6304f-66c85585fd-jtg67 namespace: kserve-ci-e2e-test resourceVersion: "21322" uid: d36fe325-e6e4-464a-876a-5eefa53603ef kind: Event lastTimestamp: "2026-09-18T16:38:54Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:38:54Z" name: ensemble-graph-6304f-66c85585fd-jtg67.18d677b82c90cb6a namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 6e91da72-6c44-4250-9c4b-ab81bd652220 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6304f} kind: Pod name: ensemble-graph-6304f-66c85585fd-jtg67 namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: d36fe325-e6e4-464a-876a-5eefa53603ef kind: Event lastTimestamp: "2026-09-18T16:38:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" already present on machine metadata: creationTimestamp: "2026-09-18T16:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:54Z" name: ensemble-graph-6304f-66c85585fd-jtg67.18d677b82dce6f85 namespace: kserve-ci-e2e-test resourceVersion: "21348" uid: a9b68897-d154-45ce-bcaa-2c535669a601 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6304f} kind: Pod name: ensemble-graph-6304f-66c85585fd-jtg67 namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: d36fe325-e6e4-464a-876a-5eefa53603ef kind: Event lastTimestamp: "2026-09-18T16:38:54Z" message: 'Created container: ensemble-graph-6304f' metadata: creationTimestamp: "2026-09-18T16:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:54Z" name: ensemble-graph-6304f-66c85585fd-jtg67.18d677b830cf1f86 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 63f5cc5f-13b4-4047-b123-a17e42384d15 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6304f} kind: Pod name: ensemble-graph-6304f-66c85585fd-jtg67 namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: d36fe325-e6e4-464a-876a-5eefa53603ef kind: Event lastTimestamp: "2026-09-18T16:38:54Z" message: Started container ensemble-graph-6304f metadata: creationTimestamp: "2026-09-18T16:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:54Z" name: ensemble-graph-6304f-66c85585fd-jtg67.18d677b8314986f4 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 8bad741a-fce8-41db-bb9c-b20bfbb505d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6304f} kind: Pod name: ensemble-graph-6304f-66c85585fd-jtg67 namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: d36fe325-e6e4-464a-876a-5eefa53603ef kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: Stopping container ensemble-graph-6304f metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:04Z" name: ensemble-graph-6304f-66c85585fd-jtg67.18d677ba5da81635 namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: 14f0dc24-5d45-4fb9-aef4-3313f57a1cb9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:39:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6304f} kind: Pod name: ensemble-graph-6304f-66c85585fd-jtg67 namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: d36fe325-e6e4-464a-876a-5eefa53603ef kind: Event lastTimestamp: "2026-09-18T16:39:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:31Z" name: ensemble-graph-6304f-66c85585fd-jtg67.18d677bae958dfae namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 424e7bfb-f0bd-4292-8dc9-63eb03350247 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-6304f-66c85585fd namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: eb911cc7-1486-4c8c-8fde-22bb446f7f11 kind: Event lastTimestamp: "2026-09-18T16:38:53Z" message: 'Created pod: ensemble-graph-6304f-66c85585fd-jtg67' metadata: creationTimestamp: "2026-09-18T16:38:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:53Z" name: ensemble-graph-6304f-66c85585fd.18d677b7e60c1042 namespace: kserve-ci-e2e-test resourceVersion: "21317" uid: 58f537ae-50fe-4235-904b-38e3e634e6f6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-6304f namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: effb94c3-ef89-4b87-805b-bedd5161c5a5 kind: Event lastTimestamp: "2026-09-18T16:38:53Z" message: Scaled up replica set ensemble-graph-6304f-66c85585fd from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:38:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:53Z" name: ensemble-graph-6304f.18d677b7e52782eb namespace: kserve-ci-e2e-test resourceVersion: "21312" uid: ba066e31-1b7e-4866-bf3f-74e4f2a9eeeb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-6304f namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: b4771636-efdf-4d09-a24f-13799eaf63d7 kind: Event lastTimestamp: "2026-09-18T16:38:53Z" message: InferenceGraph [ensemble-graph-6304f] is Ready metadata: creationTimestamp: "2026-09-18T16:38:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:53Z" name: ensemble-graph-6304f.18d677b7ebcb3ae1 namespace: kserve-ci-e2e-test resourceVersion: "21329" uid: 40a79295-d7fd-4e6d-98a2-c4128ccc3064 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:40:14.500718Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-de495-69977d94b8-6jnmn namespace: kserve-ci-e2e-test resourceVersion: "22358" uid: a1daf0a2-283d-4ee3-8687-5611c87d2083 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-de495-69977d94b8-6jnmn to ip-10-0-128-224.ec2.internal metadata: creationTimestamp: "2026-09-18T16:40:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:40:14Z" name: ensemble-graph-de495-69977d94b8-6jnmn.18d677cab9353948 namespace: kserve-ci-e2e-test resourceVersion: "22364" uid: 35bd34ea-38db-4151-955d-67b75012f8da reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:14Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-de495-69977d94b8-6jnmn namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: a1daf0a2-283d-4ee3-8687-5611c87d2083 kind: Event lastTimestamp: "2026-09-18T16:40:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-de495-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:14Z" name: ensemble-graph-de495-69977d94b8-6jnmn.18d677cac14cef74 namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: 6ac04735-9814-4cf5-9004-74c95fa2027b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:15Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-de495-69977d94b8-6jnmn namespace: kserve-ci-e2e-test resourceVersion: "22367" uid: a1daf0a2-283d-4ee3-8687-5611c87d2083 kind: Event lastTimestamp: "2026-09-18T16:40:15Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:40:15Z" name: ensemble-graph-de495-69977d94b8-6jnmn.18d677cafee956f5 namespace: kserve-ci-e2e-test resourceVersion: "22384" uid: 496bde2d-c24b-41e4-9e8e-33ee26c5670b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-de495} kind: Pod name: ensemble-graph-de495-69977d94b8-6jnmn namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: a1daf0a2-283d-4ee3-8687-5611c87d2083 kind: Event lastTimestamp: "2026-09-18T16:40:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" already present on machine metadata: creationTimestamp: "2026-09-18T16:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:15Z" name: ensemble-graph-de495-69977d94b8-6jnmn.18d677cb00187fec namespace: kserve-ci-e2e-test resourceVersion: "22386" uid: 80d883af-1810-4df7-a0c5-35917e06bc24 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-de495} kind: Pod name: ensemble-graph-de495-69977d94b8-6jnmn namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: a1daf0a2-283d-4ee3-8687-5611c87d2083 kind: Event lastTimestamp: "2026-09-18T16:40:15Z" message: 'Created container: ensemble-graph-de495' metadata: creationTimestamp: "2026-09-18T16:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:15Z" name: ensemble-graph-de495-69977d94b8-6jnmn.18d677cb05c3e403 namespace: kserve-ci-e2e-test resourceVersion: "22388" uid: 439d46c9-222f-48d7-87aa-6122a11e7cee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-de495} kind: Pod name: ensemble-graph-de495-69977d94b8-6jnmn namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: a1daf0a2-283d-4ee3-8687-5611c87d2083 kind: Event lastTimestamp: "2026-09-18T16:40:15Z" message: Started container ensemble-graph-de495 metadata: creationTimestamp: "2026-09-18T16:40:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:15Z" name: ensemble-graph-de495-69977d94b8-6jnmn.18d677cb06379b2c namespace: kserve-ci-e2e-test resourceVersion: "22389" uid: aa85ab36-42ff-4912-8b19-555fad2fdaca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-de495} kind: Pod name: ensemble-graph-de495-69977d94b8-6jnmn namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: a1daf0a2-283d-4ee3-8687-5611c87d2083 kind: Event lastTimestamp: "2026-09-18T16:48:29Z" message: Stopping container ensemble-graph-de495 metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:29Z" name: ensemble-graph-de495-69977d94b8-6jnmn.18d6783df9ae2e1f namespace: kserve-ci-e2e-test resourceVersion: "26175" uid: b84e13f1-8622-4172-a19b-fac7053ff0fd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:48:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-de495} kind: Pod name: ensemble-graph-de495-69977d94b8-6jnmn namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: a1daf0a2-283d-4ee3-8687-5611c87d2083 kind: Event lastTimestamp: "2026-09-18T16:48:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:48:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:56Z" name: ensemble-graph-de495-69977d94b8-6jnmn.18d6783e8ac25c3e namespace: kserve-ci-e2e-test resourceVersion: "26511" uid: 8010b0f6-01d8-4adb-b4eb-97ddde0e86be reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-de495-69977d94b8 namespace: kserve-ci-e2e-test resourceVersion: "22355" uid: 2a1cb8fe-f40a-4657-bf8e-1474df12c01d kind: Event lastTimestamp: "2026-09-18T16:40:14Z" message: 'Created pod: ensemble-graph-de495-69977d94b8-6jnmn' metadata: creationTimestamp: "2026-09-18T16:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:14Z" name: ensemble-graph-de495-69977d94b8.18d677cab89053db namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 711db82f-350e-4ba6-8e44-dbdf6fee76e6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-de495 namespace: kserve-ci-e2e-test resourceVersion: "22353" uid: e3062216-2184-495a-b0b6-a8e7123f9e5f kind: Event lastTimestamp: "2026-09-18T16:40:14Z" message: Scaled up replica set ensemble-graph-de495-69977d94b8 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:14Z" name: ensemble-graph-de495.18d677cab784e76e namespace: kserve-ci-e2e-test resourceVersion: "22356" uid: 93ecc6e8-83a4-4c0a-8c43-d308f0e8e439 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-de495 namespace: kserve-ci-e2e-test resourceVersion: "22369" uid: e72e50ef-bc81-4514-a4ee-9c80aca1040c kind: Event lastTimestamp: "2026-09-18T16:40:14Z" message: InferenceGraph [ensemble-graph-de495] is Ready metadata: creationTimestamp: "2026-09-18T16:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:40:14Z" name: ensemble-graph-de495.18d677cabd229d63 namespace: kserve-ci-e2e-test resourceVersion: "22371" uid: 30690243-3726-4c89-9ec8-762716602986 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:40:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-de495 namespace: kserve-ci-e2e-test resourceVersion: "22354" uid: cc324b43-5347-4c02-af10-c99a35ce9106 kind: Event lastTimestamp: "2026-09-18T16:40:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:44Z" name: ensemble-graph-de495.18d677ce35f852cf namespace: kserve-ci-e2e-test resourceVersion: "22630" uid: 4bfe0ec4-5ebc-4424-8616-4bd84cbfa2ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:40:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-de495 namespace: kserve-ci-e2e-test resourceVersion: "22354" uid: cc324b43-5347-4c02-af10-c99a35ce9106 kind: Event lastTimestamp: "2026-09-18T16:40:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:44Z" name: ensemble-graph-de495.18d677ce3664b592 namespace: kserve-ci-e2e-test resourceVersion: "22631" uid: 4025f60d-42b5-4bb6-87f3-245f2d802797 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:26:37.440525Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14926" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-191f9-predictor-86c89bb448-hmcl7 to ip-10-0-143-79.ec2.internal metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:26:37Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6770c7ca0d0f6 namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: d1f8fca5-34f9-4489-b3e5-4fca8ad1f995 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: "2026-09-18T16:26:38Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:26:38Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6770c9e531747 namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: 5eb2f563-1d1d-405a-b995-a8c529de8046 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: "2026-09-18T16:26:38Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-2013" metadata: creationTimestamp: "2026-09-18T16:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:38Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6770c9fb08c2e namespace: kserve-ci-e2e-test resourceVersion: "15015" uid: ed2ada21-bf1a-4598-ae05-3f9c593dff3a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: "2026-09-18T16:26:49Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-2013" in 11.38s (11.38s including waiting). Image size: 1384134226 bytes.' metadata: creationTimestamp: "2026-09-18T16:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:49Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6770f46088c7d namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 0fa8a330-8469-4eae-9dd3-2f8b22f64709 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: "2026-09-18T16:26:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:49Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6770f49189fa2 namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: 0dbc8a9d-3082-4793-ae55-79a92f5267c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: "2026-09-18T16:26:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:49Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6770f49855da3 namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: 705c0c33-a8e2-4ece-a685-cdd6de9bfe3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: "2026-09-18T16:26:49Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-18T16:26:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:49Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6770f499375b0 namespace: kserve-ci-e2e-test resourceVersion: "15149" uid: d033b8cd-2615-4089-86cf-42c0b5106205 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: "2026-09-18T16:26:51Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.139s (2.139s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-18T16:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:51Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6770fc9167db7 namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: 16fbe736-9b98-497c-a778-fec4f9f76c86 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: "2026-09-18T16:26:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:51Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6770fcbbf5135 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: 0e27e08a-1b9c-42c1-a3c3-b3ed3d2d1d56 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: "2026-09-18T16:26:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:26:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:51Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6770fcc302e6e namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: 20e2d40e-35c9-44c5-8ad3-51f96166a8f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T16:26:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6770ff3d9d1c1 namespace: kserve-ci-e2e-test resourceVersion: "16109" uid: 590ee2b9-e9b0-4028-9113-26332c249735 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: "2026-09-18T16:28:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:18Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6772401d3a9c4 namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: 4455fe93-e3bd-4a12-8e25-7eeacc9b43ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: "2026-09-18T16:28:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:18Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6772401d427c5 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 09d8d385-5162-4884-8288-8f17365c9ded reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2dfb1cbe-7e60-4042-9130-a95dd5ee09cb kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: error-404-isvc-191f9-predictor-86c89bb448-hmcl7.18d6772435e8232a namespace: kserve-ci-e2e-test resourceVersion: "16108" uid: 2c32f7cf-096b-4993-84d6-bb2de66f7fc8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-191f9-predictor-86c89bb448 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: 87b31547-3c9d-4cc2-9885-1015b6f8cf89 kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: 'Created pod: error-404-isvc-191f9-predictor-86c89bb448-hmcl7' metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:26:37Z" name: error-404-isvc-191f9-predictor-86c89bb448.18d6770c7c3676d9 namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: a2acd47d-c842-4036-9445-2631a13e5a3d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-191f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 0b720b8e-49a4-4625-9fd1-1db5dde55d41 kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: Scaled up replica set error-404-isvc-191f9-predictor-86c89bb448 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:26:37Z" name: error-404-isvc-191f9-predictor.18d6770c7a65b093 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 3c5b1e03-b2f4-4825-839b-f214a088740c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-191f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: fceb0362-415b-489b-99f1-27a3413ca1f6 kind: Event lastTimestamp: "2026-09-18T16:27:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:27:37Z" name: error-404-isvc-191f9-predictor.18d6770ff8edbcd1 namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: 2650987f-39de-4e9c-a015-568f595d7696 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-191f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: fceb0362-415b-489b-99f1-27a3413ca1f6 kind: Event lastTimestamp: "2026-09-18T16:27:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:27:37Z" name: error-404-isvc-191f9-predictor.18d6770ff955c393 namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: 8d6f118a-adce-444c-8038-cddedbcedd91 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:27:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-191f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "15181" uid: fceb0362-415b-489b-99f1-27a3413ca1f6 kind: Event lastTimestamp: "2026-09-18T16:28:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:07Z" name: error-404-isvc-191f9-predictor.18d6771df42027a9 namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: 74ce44f4-ab87-4a3a-9db4-f7ebd8ec6ce0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:27:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-191f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "15181" uid: fceb0362-415b-489b-99f1-27a3413ca1f6 kind: Event lastTimestamp: "2026-09-18T16:28:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:07Z" name: error-404-isvc-191f9-predictor.18d6771df4860d19 namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: 0c6c501f-1284-4314-a3d3-417dcf9299d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-191f9 namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: 09eb4f59-7dd6-433a-8b24-b827face5b29 kind: Event lastTimestamp: "2026-09-18T16:27:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:27:39Z" name: error-404-isvc-191f9.18d6770c7a667740 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 535b83d5-4e33-407e-8c94-e400e17cf708 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-191f9 namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 09eb4f59-7dd6-433a-8b24-b827face5b29 kind: Event lastTimestamp: "2026-09-18T16:27:39Z" message: InferenceService [error-404-isvc-191f9] is Ready metadata: creationTimestamp: "2026-09-18T16:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:27:39Z" name: error-404-isvc-191f9.18d6771aeb6ecb4d namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: 70846a55-4076-4a54-a0d4-a93caad04e56 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:37:43.749448Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6304f-predictor-68f6bc4694-98blq namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: dc0653d3-39ed-486a-a1cd-1d4ba878feda kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6304f-predictor-68f6bc4694-98blq to ip-10-0-143-79.ec2.internal metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:37:43Z" name: error-404-isvc-6304f-predictor-68f6bc4694-98blq.18d677a79fbb7ae9 namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: a1bd1c3c-31c2-4d97-b701-9ff06abb9a6a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6304f-predictor-68f6bc4694-98blq namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: dc0653d3-39ed-486a-a1cd-1d4ba878feda kind: Event lastTimestamp: "2026-09-18T16:37:44Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:37:44Z" name: error-404-isvc-6304f-predictor-68f6bc4694-98blq.18d677a7c17fbbb6 namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: 565f717d-586d-401a-a550-f0dad859664f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6304f-predictor-68f6bc4694-98blq namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: dc0653d3-39ed-486a-a1cd-1d4ba878feda kind: Event lastTimestamp: "2026-09-18T16:37:44Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:44Z" name: error-404-isvc-6304f-predictor-68f6bc4694-98blq.18d677a7c2e0b3f8 namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: ef4e4263-04d3-42aa-8ae9-1fa613325bcb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6304f-predictor-68f6bc4694-98blq namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: dc0653d3-39ed-486a-a1cd-1d4ba878feda kind: Event lastTimestamp: "2026-09-18T16:37:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:44Z" name: error-404-isvc-6304f-predictor-68f6bc4694-98blq.18d677a7c5f659e2 namespace: kserve-ci-e2e-test resourceVersion: "20563" uid: f05ff636-3f51-4143-b309-6dd4c49808eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6304f-predictor-68f6bc4694-98blq namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: dc0653d3-39ed-486a-a1cd-1d4ba878feda kind: Event lastTimestamp: "2026-09-18T16:37:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:44Z" name: error-404-isvc-6304f-predictor-68f6bc4694-98blq.18d677a7c65f9786 namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: f886f08c-7010-4756-8075-93101ebecbab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6304f-predictor-68f6bc4694-98blq namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: dc0653d3-39ed-486a-a1cd-1d4ba878feda kind: Event lastTimestamp: "2026-09-18T16:37:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:44Z" name: error-404-isvc-6304f-predictor-68f6bc4694-98blq.18d677a7c668d252 namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: 902a5eb7-d9cf-472f-948f-dc044ae97582 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6304f-predictor-68f6bc4694-98blq namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: dc0653d3-39ed-486a-a1cd-1d4ba878feda kind: Event lastTimestamp: "2026-09-18T16:37:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:44Z" name: error-404-isvc-6304f-predictor-68f6bc4694-98blq.18d677a7c8e903fc namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 8b8bd1fa-a976-4381-a2b6-5a9ad42a4582 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6304f-predictor-68f6bc4694-98blq namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: dc0653d3-39ed-486a-a1cd-1d4ba878feda kind: Event lastTimestamp: "2026-09-18T16:37:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:44Z" name: error-404-isvc-6304f-predictor-68f6bc4694-98blq.18d677a7c955c15c namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: 983f2330-4804-4ffa-b096-dde7e4049346 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:37:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6304f-predictor-68f6bc4694-98blq namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: dc0653d3-39ed-486a-a1cd-1d4ba878feda kind: Event lastTimestamp: "2026-09-18T16:38:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:20Z" name: error-404-isvc-6304f-predictor-68f6bc4694-98blq.18d677a7e4ff536b namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: 1b0845d0-1914-42a0-a3a9-5f20f797f344 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6304f-predictor-68f6bc4694-98blq namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: dc0653d3-39ed-486a-a1cd-1d4ba878feda kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:04Z" name: error-404-isvc-6304f-predictor-68f6bc4694-98blq.18d677ba66756c3b namespace: kserve-ci-e2e-test resourceVersion: "21490" uid: eca4c0db-9249-4f96-8754-2bc0c5681781 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6304f-predictor-68f6bc4694-98blq namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: dc0653d3-39ed-486a-a1cd-1d4ba878feda kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:04Z" name: error-404-isvc-6304f-predictor-68f6bc4694-98blq.18d677ba6675dabd namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: 7afffd5b-9cae-43cc-9b38-52b16e312862 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6304f-predictor-68f6bc4694-98blq namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: dc0653d3-39ed-486a-a1cd-1d4ba878feda kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:05Z" name: error-404-isvc-6304f-predictor-68f6bc4694-98blq.18d677bac16fd77f namespace: kserve-ci-e2e-test resourceVersion: "21576" uid: b16aab4a-5421-4718-b577-72d1bd663b56 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6304f-predictor-68f6bc4694 namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: 45cefd4c-752c-4f1c-8554-b37ba478a171 kind: Event lastTimestamp: "2026-09-18T16:37:43Z" message: 'Created pod: error-404-isvc-6304f-predictor-68f6bc4694-98blq' metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:37:43Z" name: error-404-isvc-6304f-predictor-68f6bc4694.18d677a79f4d235a namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 1e06610b-2d7b-4364-b36b-cfe3a43cfcda reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6304f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: 0f717d7d-fb7a-432c-ba8b-9161ec083172 kind: Event lastTimestamp: "2026-09-18T16:37:43Z" message: Scaled up replica set error-404-isvc-6304f-predictor-68f6bc4694 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:37:43Z" name: error-404-isvc-6304f-predictor.18d677a79d9ed16e namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: d91e0b2f-c81d-4d56-866f-03ff3b593af9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:37:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6304f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20530" uid: ed94c019-015d-423b-889f-56f0aec76794 kind: Event lastTimestamp: "2026-09-18T16:38:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:37:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:13Z" name: error-404-isvc-6304f-predictor.18d677ab1c143044 namespace: kserve-ci-e2e-test resourceVersion: "20826" uid: 888a917f-f341-47c3-b4ba-f4af0b7a0b41 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:37:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6304f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20530" uid: ed94c019-015d-423b-889f-56f0aec76794 kind: Event lastTimestamp: "2026-09-18T16:38:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:37:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:13Z" name: error-404-isvc-6304f-predictor.18d677ab1c83b456 namespace: kserve-ci-e2e-test resourceVersion: "20827" uid: 48de4def-dcbf-4fb6-b760-b94f6aafe874 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6304f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: ed94c019-015d-423b-889f-56f0aec76794 kind: Event lastTimestamp: "2026-09-18T16:38:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:43Z" name: error-404-isvc-6304f-predictor.18d677b21a520d1a namespace: kserve-ci-e2e-test resourceVersion: "21234" uid: 27df618e-ead5-48bb-967e-e11be1ff7ef3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6304f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: ed94c019-015d-423b-889f-56f0aec76794 kind: Event lastTimestamp: "2026-09-18T16:38:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:43Z" name: error-404-isvc-6304f-predictor.18d677b21ab89c95 namespace: kserve-ci-e2e-test resourceVersion: "21235" uid: c982cc9e-59f0-431f-8504-eced1426100b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:37:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6304f namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: 766a2d95-57a9-4876-a13a-f993a3c0f6fb kind: Event lastTimestamp: "2026-09-18T16:38:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:31Z" name: error-404-isvc-6304f.18d677a79da2c06c namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: 46f98107-6c74-4c25-9667-f20795421d62 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6304f namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: 766a2d95-57a9-4876-a13a-f993a3c0f6fb kind: Event lastTimestamp: "2026-09-18T16:38:31Z" message: InferenceService [error-404-isvc-6304f] is Ready metadata: creationTimestamp: "2026-09-18T16:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:31Z" name: error-404-isvc-6304f.18d677b2a4293f9f namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: 88e7e010-3443-4f01-aab9-cae9f6c997ab reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:39:46.019607Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: d1518096-00eb-41fb-96e4-d54adf968795 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-630d7-predictor-7bc45d9448-6gblb to ip-10-0-143-79.ec2.internal metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:39:46Z" name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb.18d677c4179a02bc namespace: kserve-ci-e2e-test resourceVersion: "22060" uid: 9d893df5-8f8a-495a-9ef3-54736e2c4289 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: d1518096-00eb-41fb-96e4-d54adf968795 kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-630d7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:46Z" name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb.18d677c424acad4a namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: 4e1b2f49-c4fd-4904-9809-e49a3f40909e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: d1518096-00eb-41fb-96e4-d54adf968795 kind: Event lastTimestamp: "2026-09-18T16:39:47Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:39:47Z" name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb.18d677c45d1cb2e7 namespace: kserve-ci-e2e-test resourceVersion: "22089" uid: 42b60170-f8c3-4b83-9c36-88c0fe3c3989 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: d1518096-00eb-41fb-96e4-d54adf968795 kind: Event lastTimestamp: "2026-09-18T16:39:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:47Z" name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb.18d677c45e8bbe28 namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: 203e8c74-4e57-4ebc-91c6-d92b935b9f86 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: d1518096-00eb-41fb-96e4-d54adf968795 kind: Event lastTimestamp: "2026-09-18T16:39:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:47Z" name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb.18d677c4612f29b6 namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: 83640b0e-8591-4506-8e67-95b53d9c0ea0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: d1518096-00eb-41fb-96e4-d54adf968795 kind: Event lastTimestamp: "2026-09-18T16:39:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:47Z" name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb.18d677c4619c472f namespace: kserve-ci-e2e-test resourceVersion: "22093" uid: 54bc4628-1a99-4d83-84e2-7f5ea9be46da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: d1518096-00eb-41fb-96e4-d54adf968795 kind: Event lastTimestamp: "2026-09-18T16:39:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:47Z" name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb.18d677c461a586ec namespace: kserve-ci-e2e-test resourceVersion: "22094" uid: 50fc5651-9731-4180-b8f9-b6e301b48b5d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: d1518096-00eb-41fb-96e4-d54adf968795 kind: Event lastTimestamp: "2026-09-18T16:39:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:47Z" name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb.18d677c4642fe950 namespace: kserve-ci-e2e-test resourceVersion: "22095" uid: fa15e8a8-24ff-461e-8fd2-aa329400fcdd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: d1518096-00eb-41fb-96e4-d54adf968795 kind: Event lastTimestamp: "2026-09-18T16:39:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:47Z" name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb.18d677c46498974b namespace: kserve-ci-e2e-test resourceVersion: "22096" uid: 8654640c-ebf7-431b-8033-20bed0c93032 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: d1518096-00eb-41fb-96e4-d54adf968795 kind: Event lastTimestamp: "2026-09-18T16:49:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:14Z" name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb.18d677c4a7825d9c namespace: kserve-ci-e2e-test resourceVersion: "26785" uid: 8a0bba3f-1488-431a-ad76-c1a1d3c102f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: d1518096-00eb-41fb-96e4-d54adf968795 kind: Event lastTimestamp: "2026-09-18T16:49:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:10Z" name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb.18d67847a1a704ac namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: 3c122bc3-110c-4e6f-a238-160847629f59 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: d1518096-00eb-41fb-96e4-d54adf968795 kind: Event lastTimestamp: "2026-09-18T16:49:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:10Z" name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb.18d67847a1af54df namespace: kserve-ci-e2e-test resourceVersion: "26673" uid: d614df91-3137-46da-a496-0096178de117 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: d1518096-00eb-41fb-96e4-d54adf968795 kind: Event lastTimestamp: "2026-09-18T16:49:14Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:14Z" name: error-404-isvc-630d7-predictor-7bc45d9448-6gblb.18d678487007c1d0 namespace: kserve-ci-e2e-test resourceVersion: "26784" uid: 14f35251-da2a-403f-9d67-5b97af5bb9b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-630d7-predictor-7bc45d9448 namespace: kserve-ci-e2e-test resourceVersion: "22048" uid: f6ac76db-0da6-4d27-abbc-42696467c9eb kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: 'Created pod: error-404-isvc-630d7-predictor-7bc45d9448-6gblb' metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:46Z" name: error-404-isvc-630d7-predictor-7bc45d9448.18d677c416dd8233 namespace: kserve-ci-e2e-test resourceVersion: "22057" uid: d430452b-6249-4093-b6af-d2def33667c7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-630d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22046" uid: bc75ea58-85e7-49c6-9143-7fae36170436 kind: Event lastTimestamp: "2026-09-18T16:39:45Z" message: Scaled up replica set error-404-isvc-630d7-predictor-7bc45d9448 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:45Z" name: error-404-isvc-630d7-predictor.18d677c414fc1651 namespace: kserve-ci-e2e-test resourceVersion: "22050" uid: 485be493-69ac-4331-920c-ba2c20010571 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:40:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-630d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22047" uid: 1df2b0be-9a00-41a6-8093-6acccccfd70c kind: Event lastTimestamp: "2026-09-18T16:40:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:31Z" name: error-404-isvc-630d7-predictor.18d677c79373a484 namespace: kserve-ci-e2e-test resourceVersion: "22519" uid: ee40e34a-1c36-4d0a-939f-eaacff2647ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:40:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-630d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22047" uid: 1df2b0be-9a00-41a6-8093-6acccccfd70c kind: Event lastTimestamp: "2026-09-18T16:40:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:31Z" name: error-404-isvc-630d7-predictor.18d677c793d5b227 namespace: kserve-ci-e2e-test resourceVersion: "22520" uid: b2168bf8-3522-41e3-96e3-b7254469f6f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-630d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22238" uid: 1df2b0be-9a00-41a6-8093-6acccccfd70c kind: Event lastTimestamp: "2026-09-18T16:40:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:46Z" name: error-404-isvc-630d7-predictor.18d677d21036119b namespace: kserve-ci-e2e-test resourceVersion: "22643" uid: 436d8568-07b7-412a-a0c3-e95698728abe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-630d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22238" uid: 1df2b0be-9a00-41a6-8093-6acccccfd70c kind: Event lastTimestamp: "2026-09-18T16:40:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:46Z" name: error-404-isvc-630d7-predictor.18d677d210999e35 namespace: kserve-ci-e2e-test resourceVersion: "22645" uid: e91e4eaf-9915-4f47-b57b-c1e7a24b9451 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-630d7 namespace: kserve-ci-e2e-test resourceVersion: "22034" uid: a3c16d10-82d1-4d51-a289-6f55c8107a9a kind: Event lastTimestamp: "2026-09-18T16:40:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:40:34Z" name: error-404-isvc-630d7.18d677c4152c15e4 namespace: kserve-ci-e2e-test resourceVersion: "22557" uid: bff7b057-c71d-4eab-9eed-175635e52426 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-630d7 namespace: kserve-ci-e2e-test resourceVersion: "22555" uid: a3c16d10-82d1-4d51-a289-6f55c8107a9a kind: Event lastTimestamp: "2026-09-18T16:40:34Z" message: InferenceService [error-404-isvc-630d7] is Ready metadata: creationTimestamp: "2026-09-18T16:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:40:34Z" name: error-404-isvc-630d7.18d677cf62a7ccfa namespace: kserve-ci-e2e-test resourceVersion: "22556" uid: 4e35949a-5605-4962-8b1e-cc401b371f91 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:38:24.936277Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: c533cc1e-39ac-437b-b37b-4720c41e4f3e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl to ip-10-0-143-79.ec2.internal metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:38:24Z" name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl.18d677b136a8a322 namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 3f1320be-64bb-403b-8345-196719bbbc97 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: c533cc1e-39ac-437b-b37b-4720c41e4f3e kind: Event lastTimestamp: "2026-09-18T16:38:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7c9ed-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:25Z" name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl.18d677b1449bdb78 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 8c8d2a07-cb49-434b-8eeb-403a9b9a9753 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: c533cc1e-39ac-437b-b37b-4720c41e4f3e kind: Event lastTimestamp: "2026-09-18T16:38:26Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:38:26Z" name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl.18d677b17c9d93ae namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: dec619c6-0b52-4a82-b032-0b821bd2f076 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: c533cc1e-39ac-437b-b37b-4720c41e4f3e kind: Event lastTimestamp: "2026-09-18T16:38:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:26Z" name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl.18d677b17e096448 namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: 1685b100-ecc8-47f6-8da4-1bc209bf7399 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: c533cc1e-39ac-437b-b37b-4720c41e4f3e kind: Event lastTimestamp: "2026-09-18T16:38:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:26Z" name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl.18d677b180f4b9ac namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: 7f2ad6f5-79b9-4f3d-bd83-dc304a40bebe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: c533cc1e-39ac-437b-b37b-4720c41e4f3e kind: Event lastTimestamp: "2026-09-18T16:38:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:26Z" name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl.18d677b1815a3ce3 namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: 07b03b0b-ce3a-48da-8b9d-63b51aa0a539 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: c533cc1e-39ac-437b-b37b-4720c41e4f3e kind: Event lastTimestamp: "2026-09-18T16:38:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:26Z" name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl.18d677b18163ed99 namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 793e1f7f-a56d-47ef-ab3e-3e7d6fbfbcd8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: c533cc1e-39ac-437b-b37b-4720c41e4f3e kind: Event lastTimestamp: "2026-09-18T16:38:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:26Z" name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl.18d677b184237c63 namespace: kserve-ci-e2e-test resourceVersion: "21049" uid: ca5333ad-772e-487d-a0da-5a3eddc8402d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: c533cc1e-39ac-437b-b37b-4720c41e4f3e kind: Event lastTimestamp: "2026-09-18T16:38:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:26Z" name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl.18d677b1848fa0e5 namespace: kserve-ci-e2e-test resourceVersion: "21050" uid: 139f039f-9657-4d80-b7d6-cf5906f88387 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: c533cc1e-39ac-437b-b37b-4720c41e4f3e kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:04Z" name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl.18d677b1f2a84508 namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 86e0fa80-9b37-4749-bd9d-c9c535104666 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: c533cc1e-39ac-437b-b37b-4720c41e4f3e kind: Event lastTimestamp: "2026-09-18T16:39:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:45Z" name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl.18d677c40b60aaff namespace: kserve-ci-e2e-test resourceVersion: "22012" uid: 1b09af6d-8ded-4eb3-9a79-576c753dadca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: c533cc1e-39ac-437b-b37b-4720c41e4f3e kind: Event lastTimestamp: "2026-09-18T16:39:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:45Z" name: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl.18d677c40b615053 namespace: kserve-ci-e2e-test resourceVersion: "22013" uid: d040c99e-222c-4873-a09a-848776771201 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7c9ed-predictor-c4c8dc488 namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: 1261b7dc-4b43-489e-b1ab-ead4ce66cd19 kind: Event lastTimestamp: "2026-09-18T16:38:24Z" message: 'Created pod: error-404-isvc-7c9ed-predictor-c4c8dc488-hxjcl' metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:24Z" name: error-404-isvc-7c9ed-predictor-c4c8dc488.18d677b136165c89 namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: 285d6db4-1f00-4bd5-a72d-00c211c70ab0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7c9ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: 8584d230-bf10-4536-8c24-9ca0458f1190 kind: Event lastTimestamp: "2026-09-18T16:38:24Z" message: Scaled up replica set error-404-isvc-7c9ed-predictor-c4c8dc488 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:24Z" name: error-404-isvc-7c9ed-predictor.18d677b132e11ffb namespace: kserve-ci-e2e-test resourceVersion: "20998" uid: fc6cef11-b58e-4148-bb65-7d187f482b00 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:38:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c9ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: 4fa8f6fd-4561-4b76-b5dc-a095e37a08ba kind: Event lastTimestamp: "2026-09-18T16:39:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:09Z" name: error-404-isvc-7c9ed-predictor.18d677b4b131e1e6 namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: f90106ca-2d56-4aa9-8ee6-f125a7ffa364 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:38:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c9ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: 4fa8f6fd-4561-4b76-b5dc-a095e37a08ba kind: Event lastTimestamp: "2026-09-18T16:39:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:09Z" name: error-404-isvc-7c9ed-predictor.18d677b4b16e7a48 namespace: kserve-ci-e2e-test resourceVersion: "21628" uid: 6f6b5289-2d88-4919-bebe-a650ae150aaf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:38:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c9ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: 4fa8f6fd-4561-4b76-b5dc-a095e37a08ba kind: Event lastTimestamp: "2026-09-18T16:39:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:39Z" name: error-404-isvc-7c9ed-predictor.18d677b8303433fa namespace: kserve-ci-e2e-test resourceVersion: "21912" uid: a387a05d-2acb-43bf-8dfa-8890d027640b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:38:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c9ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: 4fa8f6fd-4561-4b76-b5dc-a095e37a08ba kind: Event lastTimestamp: "2026-09-18T16:39:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:39Z" name: error-404-isvc-7c9ed-predictor.18d677b83081c9a4 namespace: kserve-ci-e2e-test resourceVersion: "21913" uid: 41201136-24d1-4d83-a519-2649c94165b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c9ed namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 6a453028-a26b-44da-8734-2b0749e1d10a kind: Event lastTimestamp: "2026-09-18T16:39:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:14Z" name: error-404-isvc-7c9ed.18d677b132a149ac namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: b99e19fe-28ff-46b6-b3d4-b0310e7e877d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c9ed namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 6a453028-a26b-44da-8734-2b0749e1d10a kind: Event lastTimestamp: "2026-09-18T16:38:24Z" message: 'Failed to update status for InferenceService "error-404-isvc-7c9ed": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7c9ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:24Z" name: error-404-isvc-7c9ed.18d677b133881698 namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: 76304c5a-a563-464f-8253-cfe81c3ea5e8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c9ed namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 6a453028-a26b-44da-8734-2b0749e1d10a kind: Event lastTimestamp: "2026-09-18T16:38:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7c9ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:24Z" name: error-404-isvc-7c9ed.18d677b133888d5a namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: 8ba19493-fd6d-4d73-887b-ae936aa5dbd0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c9ed namespace: kserve-ci-e2e-test resourceVersion: "21663" uid: 6a453028-a26b-44da-8734-2b0749e1d10a kind: Event lastTimestamp: "2026-09-18T16:39:14Z" message: InferenceService [error-404-isvc-7c9ed] is Ready metadata: creationTimestamp: "2026-09-18T16:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:14Z" name: error-404-isvc-7c9ed.18d677bcae9aaf30 namespace: kserve-ci-e2e-test resourceVersion: "21664" uid: 08a6325f-ecaf-4bd8-b6a2-b4cd47d5c204 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:28:18.658258Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 0072adb6-9ad1-44b4-a14c-c15b5c19466c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8aa04-predictor-84fd66b648-4gjpf to ip-10-0-143-79.ec2.internal metadata: creationTimestamp: "2026-09-18T16:28:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:28:18Z" name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf.18d677240dac37cb namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: cd29d54c-7e98-41fa-b2d8-f29e2ffa55ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 0072adb6-9ad1-44b4-a14c-c15b5c19466c kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:28:19Z" name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf.18d677242f8bb5b0 namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: ad05ed4c-c12f-455f-9c1c-b0dad086d0f8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 0072adb6-9ad1-44b4-a14c-c15b5c19466c kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf.18d6772430d519d2 namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: 7d46450d-d5e2-4c81-8806-a7c48e8ef670 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 0072adb6-9ad1-44b4-a14c-c15b5c19466c kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf.18d67724335ddeaa namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: 4d86df12-615c-45da-b18d-4fd4e0c6e76f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 0072adb6-9ad1-44b4-a14c-c15b5c19466c kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf.18d6772433ba1ca0 namespace: kserve-ci-e2e-test resourceVersion: "16106" uid: 6d7e1302-097b-4c55-860a-b3b0ad6b6d85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 0072adb6-9ad1-44b4-a14c-c15b5c19466c kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf.18d6772433c3067b namespace: kserve-ci-e2e-test resourceVersion: "16107" uid: 6228bbc4-98f5-434f-a9a8-930fb19fc3df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 0072adb6-9ad1-44b4-a14c-c15b5c19466c kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf.18d6772436aa356d namespace: kserve-ci-e2e-test resourceVersion: "16110" uid: 58696c25-d122-4fcf-ae98-bc438526275b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 0072adb6-9ad1-44b4-a14c-c15b5c19466c kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf.18d6772437072a09 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: dbb0b569-641e-4198-bcf2-36fc04f69181 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 0072adb6-9ad1-44b4-a14c-c15b5c19466c kind: Event lastTimestamp: "2026-09-18T16:37:45Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:45Z" name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf.18d677244ad39a6f namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 20161059-4686-44c8-bbb7-0c5f16f271b6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 0072adb6-9ad1-44b4-a14c-c15b5c19466c kind: Event lastTimestamp: "2026-09-18T16:37:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:43Z" name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf.18d677a7949c4025 namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 275ce462-0c85-44e5-bdc8-d8a85ce8787c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 0072adb6-9ad1-44b4-a14c-c15b5c19466c kind: Event lastTimestamp: "2026-09-18T16:37:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:43Z" name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf.18d677a7949cb0ae namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: 70ea440f-7120-48a3-8ad8-5f5be279171e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 0072adb6-9ad1-44b4-a14c-c15b5c19466c kind: Event lastTimestamp: "2026-09-18T16:37:45Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:45Z" name: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf.18d677a8133d7fc4 namespace: kserve-ci-e2e-test resourceVersion: "20590" uid: ab21c380-81ff-47a7-a45f-e530b1b39e18 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8aa04-predictor-84fd66b648 namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: f5b9c2f5-a374-4010-829d-96a2e79e918c kind: Event lastTimestamp: "2026-09-18T16:28:18Z" message: 'Created pod: error-404-isvc-8aa04-predictor-84fd66b648-4gjpf' metadata: creationTimestamp: "2026-09-18T16:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:18Z" name: error-404-isvc-8aa04-predictor-84fd66b648.18d677240d3941a2 namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: f9824b48-b6b0-4140-ac36-fa959118b12e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8aa04-predictor namespace: kserve-ci-e2e-test resourceVersion: "16060" uid: 61c06996-9e3d-47ea-9978-7b64d835673e kind: Event lastTimestamp: "2026-09-18T16:28:18Z" message: Scaled up replica set error-404-isvc-8aa04-predictor-84fd66b648 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:18Z" name: error-404-isvc-8aa04-predictor.18d677240b9c0221 namespace: kserve-ci-e2e-test resourceVersion: "16064" uid: 76cc818f-cf0e-4807-bb49-728f3910ad74 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:28:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8aa04-predictor namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: cd174458-6161-46f8-9870-786c86e091e3 kind: Event lastTimestamp: "2026-09-18T16:28:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:48Z" name: error-404-isvc-8aa04-predictor.18d677278a13977c namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 7a8df932-8ff8-477f-ab64-f8e037735cfb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:28:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8aa04-predictor namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: cd174458-6161-46f8-9870-786c86e091e3 kind: Event lastTimestamp: "2026-09-18T16:28:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:48Z" name: error-404-isvc-8aa04-predictor.18d677278a600962 namespace: kserve-ci-e2e-test resourceVersion: "16428" uid: ba3bd8c9-ad02-4ecf-b945-027bf70f8df3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:29:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8aa04-predictor namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: cd174458-6161-46f8-9870-786c86e091e3 kind: Event lastTimestamp: "2026-09-18T16:29:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:29:33Z" name: error-404-isvc-8aa04-predictor.18d6772e87d9341b namespace: kserve-ci-e2e-test resourceVersion: "17018" uid: 20a68a2a-071f-4a8e-9a76-77e51acf0048 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:29:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8aa04-predictor namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: cd174458-6161-46f8-9870-786c86e091e3 kind: Event lastTimestamp: "2026-09-18T16:29:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:29:33Z" name: error-404-isvc-8aa04-predictor.18d6772e88435673 namespace: kserve-ci-e2e-test resourceVersion: "17019" uid: 974c4981-35da-40bf-a940-fe77f2035eca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:28:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8aa04 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: f230e943-54ea-4c68-9b58-7f564b42edca kind: Event lastTimestamp: "2026-09-18T16:29:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:29:05Z" name: error-404-isvc-8aa04.18d677240ba019bd namespace: kserve-ci-e2e-test resourceVersion: "16753" uid: 3e0f127d-b128-4603-a2b1-fe02cd6379b6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8aa04 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: f230e943-54ea-4c68-9b58-7f564b42edca kind: Event lastTimestamp: "2026-09-18T16:29:05Z" message: InferenceService [error-404-isvc-8aa04] is Ready metadata: creationTimestamp: "2026-09-18T16:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:29:05Z" name: error-404-isvc-8aa04.18d6772f074b4c64 namespace: kserve-ci-e2e-test resourceVersion: "16750" uid: 8abce264-49f8-4861-9876-c0a1391e1e80 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:28:59.250284Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974 namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: 55f2e7a0-99e3-40c7-9f66-e3069b10a4b9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8d780-predictor-7c7b685fcd-6l974 to ip-10-0-143-79.ec2.internal metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:28:59Z" name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974.18d6772d812562c9 namespace: kserve-ci-e2e-test resourceVersion: "16642" uid: 4e1115bf-76c6-4c7e-bf8f-2f3db120bc24 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974 namespace: kserve-ci-e2e-test resourceVersion: "16648" uid: 55f2e7a0-99e3-40c7-9f66-e3069b10a4b9 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:28:59Z" name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974.18d6772da3252642 namespace: kserve-ci-e2e-test resourceVersion: "16665" uid: b565bc35-6ba5-41bf-b777-9c5ce43b7c8b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 55f2e7a0-99e3-40c7-9f66-e3069b10a4b9 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:59Z" name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974.18d6772da48d7044 namespace: kserve-ci-e2e-test resourceVersion: "16667" uid: b0f18c05-03b6-495a-8238-1248392cd645 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 55f2e7a0-99e3-40c7-9f66-e3069b10a4b9 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:59Z" name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974.18d6772da7a42ec5 namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: 0a884a39-e12f-452d-acb6-9bafe4370944 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 55f2e7a0-99e3-40c7-9f66-e3069b10a4b9 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:59Z" name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974.18d6772da80aa774 namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: 9e176e62-cb2d-4a40-a88b-66647a29f09f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 55f2e7a0-99e3-40c7-9f66-e3069b10a4b9 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:59Z" name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974.18d6772da813cbeb namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: 8c03a54e-4b60-46ae-8915-9418c9fff8ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 55f2e7a0-99e3-40c7-9f66-e3069b10a4b9 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:59Z" name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974.18d6772daa5f78d3 namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: ca356250-c2b8-4a89-b2cb-f4047659f10a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 55f2e7a0-99e3-40c7-9f66-e3069b10a4b9 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:59Z" name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974.18d6772daac58a4c namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: 091fb725-2f99-4e1a-9165-bf463baeb50b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:29:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 55f2e7a0-99e3-40c7-9f66-e3069b10a4b9 kind: Event lastTimestamp: "2026-09-18T16:38:26Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:29:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:26Z" name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974.18d6772de01eca7f namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: 06e08ca5-2b61-4fbc-b6d1-9aa097cbd4d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 55f2e7a0-99e3-40c7-9f66-e3069b10a4b9 kind: Event lastTimestamp: "2026-09-18T16:38:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:24Z" name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974.18d677b1280c8dfa namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 8a7c6ec5-ff6a-4307-8dd3-a73d640f6539 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 55f2e7a0-99e3-40c7-9f66-e3069b10a4b9 kind: Event lastTimestamp: "2026-09-18T16:38:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:24Z" name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974.18d677b1280d1941 namespace: kserve-ci-e2e-test resourceVersion: "20970" uid: 8b8d0700-b95f-43ed-bc51-7d5d80b423c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 55f2e7a0-99e3-40c7-9f66-e3069b10a4b9 kind: Event lastTimestamp: "2026-09-18T16:38:26Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:26Z" name: error-404-isvc-8d780-predictor-7c7b685fcd-6l974.18d677b1a87da676 namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: b25e9cd4-1c91-4836-9c03-74f6539188d6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8d780-predictor-7c7b685fcd namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 9c7acf27-140e-4151-988b-c0e618faf55b kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: 'Created pod: error-404-isvc-8d780-predictor-7c7b685fcd-6l974' metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:59Z" name: error-404-isvc-8d780-predictor-7c7b685fcd.18d6772d80b362c5 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 8e3a2fd2-474e-4a5f-8606-e6eeb9e5e20f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8d780-predictor namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 4241c534-3bd5-4155-93d6-c6e4b5d9021c kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: Scaled up replica set error-404-isvc-8d780-predictor-7c7b685fcd from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:59Z" name: error-404-isvc-8d780-predictor.18d6772d7de5ec3e namespace: kserve-ci-e2e-test resourceVersion: "16631" uid: b2c9aafe-5876-4544-9d5e-b9e7929cc554 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:29:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d780-predictor namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: 53c90196-850e-41d6-a341-7b15e6132939 kind: Event lastTimestamp: "2026-09-18T16:29:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:29:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:29:29Z" name: error-404-isvc-8d780-predictor.18d67730fb14c64f namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: d8ada7a2-1cf4-48b9-a63b-f6d82cb5e626 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:29:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d780-predictor namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: 53c90196-850e-41d6-a341-7b15e6132939 kind: Event lastTimestamp: "2026-09-18T16:29:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:29:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:29:29Z" name: error-404-isvc-8d780-predictor.18d67730fb826860 namespace: kserve-ci-e2e-test resourceVersion: "16973" uid: 390a6e8d-97c3-46ad-915f-a084da10a860 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:29:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d780-predictor namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 53c90196-850e-41d6-a341-7b15e6132939 kind: Event lastTimestamp: "2026-09-18T16:30:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:30:14Z" name: error-404-isvc-8d780-predictor.18d67737f8e2f19d namespace: kserve-ci-e2e-test resourceVersion: "17377" uid: e2965573-35af-4a7f-a205-6b29e9ee526c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:29:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d780-predictor namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: 53c90196-850e-41d6-a341-7b15e6132939 kind: Event lastTimestamp: "2026-09-18T16:30:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:30:14Z" name: error-404-isvc-8d780-predictor.18d67737f9567cc3 namespace: kserve-ci-e2e-test resourceVersion: "17378" uid: 4d51815c-d442-41a1-a714-73f632c311f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d780 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 8b66eda0-d6e0-44d2-9236-56edaec3c023 kind: Event lastTimestamp: "2026-09-18T16:29:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:29:46Z" name: error-404-isvc-8d780.18d6772d7c7943dc namespace: kserve-ci-e2e-test resourceVersion: "17133" uid: 32cfb3ae-1d7f-451d-a6cf-cea4152cb5a5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d780 namespace: kserve-ci-e2e-test resourceVersion: "17131" uid: 8b66eda0-d6e0-44d2-9236-56edaec3c023 kind: Event lastTimestamp: "2026-09-18T16:29:46Z" message: InferenceService [error-404-isvc-8d780] is Ready metadata: creationTimestamp: "2026-09-18T16:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:29:46Z" name: error-404-isvc-8d780.18d677389c10e0d1 namespace: kserve-ci-e2e-test resourceVersion: "17132" uid: a9fa8ee0-7c67-4b05-9e44-84b20f6bd69a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:49:11.192047Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a2696-predictor-548b77d64-8zrh5 namespace: kserve-ci-e2e-test resourceVersion: "26714" uid: 2686ece5-1634-4763-9774-a5eb35d7cbb4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a2696-predictor-548b77d64-8zrh5 to ip-10-0-143-79.ec2.internal metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:49:11Z" name: error-404-isvc-a2696-predictor-548b77d64-8zrh5.18d67847ae811733 namespace: kserve-ci-e2e-test resourceVersion: "26718" uid: dcfc38bc-6ad5-4f7d-a69c-f49991a1bd0c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a2696-predictor-548b77d64-8zrh5 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 2686ece5-1634-4763-9774-a5eb35d7cbb4 kind: Event lastTimestamp: "2026-09-18T16:49:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a2696-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:11Z" name: error-404-isvc-a2696-predictor-548b77d64-8zrh5.18d67847b7dcf356 namespace: kserve-ci-e2e-test resourceVersion: "26730" uid: e4fcc708-4196-499d-8cbe-112281c92248 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a2696-predictor-548b77d64-8zrh5 namespace: kserve-ci-e2e-test resourceVersion: "26724" uid: 2686ece5-1634-4763-9774-a5eb35d7cbb4 kind: Event lastTimestamp: "2026-09-18T16:49:12Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:49:12Z" name: error-404-isvc-a2696-predictor-548b77d64-8zrh5.18d67847f485ccf8 namespace: kserve-ci-e2e-test resourceVersion: "26752" uid: 41f3f170-0fa7-437a-a222-709379837406 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a2696-predictor-548b77d64-8zrh5 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 2686ece5-1634-4763-9774-a5eb35d7cbb4 kind: Event lastTimestamp: "2026-09-18T16:49:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:12Z" name: error-404-isvc-a2696-predictor-548b77d64-8zrh5.18d67847f60b343e namespace: kserve-ci-e2e-test resourceVersion: "26754" uid: 544a4265-b359-4182-84f8-1ac1dbb76e8e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a2696-predictor-548b77d64-8zrh5 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 2686ece5-1634-4763-9774-a5eb35d7cbb4 kind: Event lastTimestamp: "2026-09-18T16:49:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:12Z" name: error-404-isvc-a2696-predictor-548b77d64-8zrh5.18d67847fa0e0e0a namespace: kserve-ci-e2e-test resourceVersion: "26756" uid: 0665121a-bbb5-4b71-ad62-3e2fd497cba0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a2696-predictor-548b77d64-8zrh5 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 2686ece5-1634-4763-9774-a5eb35d7cbb4 kind: Event lastTimestamp: "2026-09-18T16:49:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:12Z" name: error-404-isvc-a2696-predictor-548b77d64-8zrh5.18d67847fa9052cb namespace: kserve-ci-e2e-test resourceVersion: "26757" uid: 587d6188-a8d9-4995-bb73-1194c68628e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a2696-predictor-548b77d64-8zrh5 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 2686ece5-1634-4763-9774-a5eb35d7cbb4 kind: Event lastTimestamp: "2026-09-18T16:49:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:12Z" name: error-404-isvc-a2696-predictor-548b77d64-8zrh5.18d67847fa9dad44 namespace: kserve-ci-e2e-test resourceVersion: "26758" uid: f54b463d-428e-492f-a2b7-c815ff51220c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a2696-predictor-548b77d64-8zrh5 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 2686ece5-1634-4763-9774-a5eb35d7cbb4 kind: Event lastTimestamp: "2026-09-18T16:49:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:12Z" name: error-404-isvc-a2696-predictor-548b77d64-8zrh5.18d67847fdca2ca2 namespace: kserve-ci-e2e-test resourceVersion: "26759" uid: 533a7a6b-c937-439e-a39c-ac0aa7168bae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a2696-predictor-548b77d64-8zrh5 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 2686ece5-1634-4763-9774-a5eb35d7cbb4 kind: Event lastTimestamp: "2026-09-18T16:49:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:12Z" name: error-404-isvc-a2696-predictor-548b77d64-8zrh5.18d67847fe4d31de namespace: kserve-ci-e2e-test resourceVersion: "26760" uid: d835e57b-d45d-4f53-898c-9c018c3545b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:49:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a2696-predictor-548b77d64-8zrh5 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 2686ece5-1634-4763-9774-a5eb35d7cbb4 kind: Event lastTimestamp: "2026-09-18T16:49:59Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:49:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:59Z" name: error-404-isvc-a2696-predictor-548b77d64-8zrh5.18d6784847475969 namespace: kserve-ci-e2e-test resourceVersion: "27364" uid: 9ac097f4-9545-448c-bb8b-9f72244340f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a2696-predictor-548b77d64-8zrh5 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 2686ece5-1634-4763-9774-a5eb35d7cbb4 kind: Event lastTimestamp: "2026-09-18T17:06:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:41Z" name: error-404-isvc-a2696-predictor-548b77d64-8zrh5.18d6793c2d850b26 namespace: kserve-ci-e2e-test resourceVersion: "34499" uid: e4915f4a-26c2-4dc6-a894-64d65b863566 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a2696-predictor-548b77d64-8zrh5 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 2686ece5-1634-4763-9774-a5eb35d7cbb4 kind: Event lastTimestamp: "2026-09-18T17:06:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:41Z" name: error-404-isvc-a2696-predictor-548b77d64-8zrh5.18d6793c2d8579f1 namespace: kserve-ci-e2e-test resourceVersion: "34500" uid: dcef3068-2482-455e-be3e-ee88b09a3522 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a2696-predictor-548b77d64 namespace: kserve-ci-e2e-test resourceVersion: "26708" uid: d2a4023a-ec67-45fe-9a78-cdac7e6c3aa9 kind: Event lastTimestamp: "2026-09-18T16:49:11Z" message: 'Created pod: error-404-isvc-a2696-predictor-548b77d64-8zrh5' metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:11Z" name: error-404-isvc-a2696-predictor-548b77d64.18d67847ae22d001 namespace: kserve-ci-e2e-test resourceVersion: "26716" uid: a7fdd506-d7de-4aed-9287-f860d1e437d0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a2696-predictor namespace: kserve-ci-e2e-test resourceVersion: "26705" uid: 1c12c8af-b878-4a64-94d7-e16729e67059 kind: Event lastTimestamp: "2026-09-18T16:49:11Z" message: Scaled up replica set error-404-isvc-a2696-predictor-548b77d64 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:11Z" name: error-404-isvc-a2696-predictor.18d67847abe1f53d namespace: kserve-ci-e2e-test resourceVersion: "26709" uid: 04e5a022-ff4b-468e-b253-ebc409cfc7ae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:49:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a2696-predictor namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 1d54289f-23b0-4949-87c4-fcd11707d19b kind: Event lastTimestamp: "2026-09-18T16:49:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:41Z" name: error-404-isvc-a2696-predictor.18d6784b2a506976 namespace: kserve-ci-e2e-test resourceVersion: "27057" uid: 39811f4f-c43d-4531-bf62-1a87acf63992 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:49:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a2696-predictor namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 1d54289f-23b0-4949-87c4-fcd11707d19b kind: Event lastTimestamp: "2026-09-18T16:49:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:41Z" name: error-404-isvc-a2696-predictor.18d6784b2ab01490 namespace: kserve-ci-e2e-test resourceVersion: "27058" uid: fefa5398-e5df-4cc6-a18a-2e0fd1426b23 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:49:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a2696-predictor namespace: kserve-ci-e2e-test resourceVersion: "26885" uid: 1d54289f-23b0-4949-87c4-fcd11707d19b kind: Event lastTimestamp: "2026-09-18T16:50:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:50:26Z" name: error-404-isvc-a2696-predictor.18d6785228303675 namespace: kserve-ci-e2e-test resourceVersion: "27631" uid: 10d56549-3d22-4af0-9aa3-6fc0bc0f68d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:49:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a2696-predictor namespace: kserve-ci-e2e-test resourceVersion: "26885" uid: 1d54289f-23b0-4949-87c4-fcd11707d19b kind: Event lastTimestamp: "2026-09-18T16:50:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:50:26Z" name: error-404-isvc-a2696-predictor.18d678522899b106 namespace: kserve-ci-e2e-test resourceVersion: "27632" uid: 201f867a-da87-4c2d-9936-c0b53952caf9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a2696 namespace: kserve-ci-e2e-test resourceVersion: "26693" uid: 8540b184-1bfc-459e-a03e-ca3a693aa3ff kind: Event lastTimestamp: "2026-09-18T16:50:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:50:09Z" name: error-404-isvc-a2696.18d67847abf9e365 namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: e5e0a579-ecab-4fab-8ac8-7f7f0c764a4e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:50:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a2696 namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 8540b184-1bfc-459e-a03e-ca3a693aa3ff kind: Event lastTimestamp: "2026-09-18T16:50:09Z" message: InferenceService [error-404-isvc-a2696] is Ready metadata: creationTimestamp: "2026-09-18T16:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:50:09Z" name: error-404-isvc-a2696.18d67855592a064f namespace: kserve-ci-e2e-test resourceVersion: "27459" uid: a9bb37bb-3416-4cce-9799-def65aa542fc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:48:29.789315Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf namespace: kserve-ci-e2e-test resourceVersion: "26256" uid: 09d20b97-e9ed-4b18-8463-e3dfe8f907a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b3de6-predictor-789679b4dd-5cfbf to ip-10-0-143-79.ec2.internal metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:48:29Z" name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf.18d6783e0ab5baf3 namespace: kserve-ci-e2e-test resourceVersion: "26260" uid: 6ac700bf-04e9-4b5e-ba32-cf6792d9319f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf namespace: kserve-ci-e2e-test resourceVersion: "26266" uid: 09d20b97-e9ed-4b18-8463-e3dfe8f907a1 kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:48:30Z" name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf.18d6783e2d734b86 namespace: kserve-ci-e2e-test resourceVersion: "26282" uid: 7d71ff2d-649b-4a11-a8eb-74ff31090537 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 09d20b97-e9ed-4b18-8463-e3dfe8f907a1 kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:30Z" name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf.18d6783e2fcf4700 namespace: kserve-ci-e2e-test resourceVersion: "26286" uid: 03962caf-d033-4ddb-ba3b-87553ab42322 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 09d20b97-e9ed-4b18-8463-e3dfe8f907a1 kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:30Z" name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf.18d6783e33981d1a namespace: kserve-ci-e2e-test resourceVersion: "26290" uid: efa29012-ca39-4ce3-9d18-69dcf234648a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 09d20b97-e9ed-4b18-8463-e3dfe8f907a1 kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:30Z" name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf.18d6783e34212d82 namespace: kserve-ci-e2e-test resourceVersion: "26291" uid: 162c6a0f-ca34-492a-8b7c-43cf97fa0fa5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 09d20b97-e9ed-4b18-8463-e3dfe8f907a1 kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:30Z" name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf.18d6783e342edb59 namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: f9d726fe-f9a9-4256-8629-e713fa1f8f82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 09d20b97-e9ed-4b18-8463-e3dfe8f907a1 kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:30Z" name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf.18d6783e37151100 namespace: kserve-ci-e2e-test resourceVersion: "26293" uid: 48f9030a-553b-468d-8bde-38ac21201082 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 09d20b97-e9ed-4b18-8463-e3dfe8f907a1 kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:30Z" name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf.18d6783e3796e955 namespace: kserve-ci-e2e-test resourceVersion: "26294" uid: e9476175-6441-4595-ad9e-80356ef39321 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:48:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 09d20b97-e9ed-4b18-8463-e3dfe8f907a1 kind: Event lastTimestamp: "2026-09-18T16:49:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:48:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:17Z" name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf.18d6783e74086626 namespace: kserve-ci-e2e-test resourceVersion: "26822" uid: 517d6913-dfa1-4333-8f11-41f00ecdd32a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 09d20b97-e9ed-4b18-8463-e3dfe8f907a1 kind: Event lastTimestamp: "2026-09-18T16:49:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:50Z" name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf.18d67850ef162e38 namespace: kserve-ci-e2e-test resourceVersion: "27191" uid: c498bd75-2312-4f4e-9a2b-8f2b8b54f390 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 09d20b97-e9ed-4b18-8463-e3dfe8f907a1 kind: Event lastTimestamp: "2026-09-18T16:49:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:50Z" name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf.18d67850ef174979 namespace: kserve-ci-e2e-test resourceVersion: "27192" uid: 6610f891-a4f6-49bd-aabd-830907c62777 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 09d20b97-e9ed-4b18-8463-e3dfe8f907a1 kind: Event lastTimestamp: "2026-09-18T16:49:52Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": dial tcp 10.132.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:52Z" name: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf.18d678515036bf59 namespace: kserve-ci-e2e-test resourceVersion: "27282" uid: f002b10b-acaf-4669-82b3-bc1c43146876 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b3de6-predictor-789679b4dd namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: faedefad-cba7-4a47-915d-751bfaed7f31 kind: Event lastTimestamp: "2026-09-18T16:48:29Z" message: 'Created pod: error-404-isvc-b3de6-predictor-789679b4dd-5cfbf' metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:29Z" name: error-404-isvc-b3de6-predictor-789679b4dd.18d6783e0a52e1e2 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 2b86de66-0dc1-4955-ab8b-a5d59d69a904 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b3de6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 03b1019a-7031-40cd-88e7-989152600237 kind: Event lastTimestamp: "2026-09-18T16:48:29Z" message: Scaled up replica set error-404-isvc-b3de6-predictor-789679b4dd from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:29Z" name: error-404-isvc-b3de6-predictor.18d6783e08a69e43 namespace: kserve-ci-e2e-test resourceVersion: "26250" uid: cd50295e-215f-49cc-9f55-d16640aca632 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:48:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b3de6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26247" uid: 44600a11-9635-498d-9354-b9e01049469e kind: Event lastTimestamp: "2026-09-18T16:48:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:59Z" name: error-404-isvc-b3de6-predictor.18d678418730c04c namespace: kserve-ci-e2e-test resourceVersion: "26534" uid: 75cf3603-106e-43c7-aac0-09a3e4fa6e08 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:48:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b3de6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26247" uid: 44600a11-9635-498d-9354-b9e01049469e kind: Event lastTimestamp: "2026-09-18T16:48:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:59Z" name: error-404-isvc-b3de6-predictor.18d67841879a1b2f namespace: kserve-ci-e2e-test resourceVersion: "26535" uid: 7194da12-3661-442d-a0d6-b7306c0ec72e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:49:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b3de6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 44600a11-9635-498d-9354-b9e01049469e kind: Event lastTimestamp: "2026-09-18T16:49:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:44Z" name: error-404-isvc-b3de6-predictor.18d67848851a0048 namespace: kserve-ci-e2e-test resourceVersion: "27090" uid: c812560e-917b-4c68-baa0-8cd5e45cf2fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:49:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b3de6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 44600a11-9635-498d-9354-b9e01049469e kind: Event lastTimestamp: "2026-09-18T16:49:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:44Z" name: error-404-isvc-b3de6-predictor.18d67848858653bb namespace: kserve-ci-e2e-test resourceVersion: "27091" uid: 32933e00-52f2-431f-b337-50d61f66d582 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:48:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b3de6 namespace: kserve-ci-e2e-test resourceVersion: "26216" uid: 061a07a0-340f-453c-abb0-b9384b93af00 kind: Event lastTimestamp: "2026-09-18T16:49:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:49:27Z" name: error-404-isvc-b3de6.18d6783e08b86643 namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: e72805b1-dfb1-480f-9020-59f9506916eb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b3de6 namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: 061a07a0-340f-453c-abb0-b9384b93af00 kind: Event lastTimestamp: "2026-09-18T16:49:27Z" message: InferenceService [error-404-isvc-b3de6] is Ready metadata: creationTimestamp: "2026-09-18T16:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:49:27Z" name: error-404-isvc-b3de6.18d6784b83998778 namespace: kserve-ci-e2e-test resourceVersion: "26915" uid: 32210c6c-d489-4134-a085-996e9b0d6af0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:49:51.117275Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t namespace: kserve-ci-e2e-test resourceVersion: "27231" uid: 1fdb4863-f39d-41db-bf3c-1b04a45ac05b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bb388-predictor-6d44f64874-nwx7t to ip-10-0-143-79.ec2.internal metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:49:51Z" name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t.18d67850fa3bb920 namespace: kserve-ci-e2e-test resourceVersion: "27235" uid: d71dfe48-472c-46f2-a9d2-9d2941bbaefb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 1fdb4863-f39d-41db-bf3c-1b04a45ac05b kind: Event lastTimestamp: "2026-09-18T16:49:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-bb388-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:51Z" name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t.18d67851054677bd namespace: kserve-ci-e2e-test resourceVersion: "27248" uid: c4bac362-093d-4c27-b822-36df687746d0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t namespace: kserve-ci-e2e-test resourceVersion: "27240" uid: 1fdb4863-f39d-41db-bf3c-1b04a45ac05b kind: Event lastTimestamp: "2026-09-18T16:49:52Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:49:52Z" name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t.18d6785140728bf1 namespace: kserve-ci-e2e-test resourceVersion: "27272" uid: ce2aba8c-b809-4dc6-b300-fa118f56090c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 1fdb4863-f39d-41db-bf3c-1b04a45ac05b kind: Event lastTimestamp: "2026-09-18T16:49:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:52Z" name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t.18d6785141e28338 namespace: kserve-ci-e2e-test resourceVersion: "27274" uid: e1a96243-946c-4bf6-86d1-b1049f235a9b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 1fdb4863-f39d-41db-bf3c-1b04a45ac05b kind: Event lastTimestamp: "2026-09-18T16:49:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:52Z" name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t.18d6785144e903c3 namespace: kserve-ci-e2e-test resourceVersion: "27275" uid: 4e731645-b342-40b4-9f47-081275987f9f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 1fdb4863-f39d-41db-bf3c-1b04a45ac05b kind: Event lastTimestamp: "2026-09-18T16:49:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:52Z" name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t.18d67851456bb05a namespace: kserve-ci-e2e-test resourceVersion: "27277" uid: 81a3a976-b6c3-4f14-8ab8-99da47921b7f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 1fdb4863-f39d-41db-bf3c-1b04a45ac05b kind: Event lastTimestamp: "2026-09-18T16:49:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:52Z" name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t.18d6785145795c6f namespace: kserve-ci-e2e-test resourceVersion: "27278" uid: 6c3abc7d-6654-4bc5-b005-b31c347610b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 1fdb4863-f39d-41db-bf3c-1b04a45ac05b kind: Event lastTimestamp: "2026-09-18T16:49:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:52Z" name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t.18d6785149575a84 namespace: kserve-ci-e2e-test resourceVersion: "27279" uid: 0c0fb483-8856-4a5d-8000-ad84afac4eb8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 1fdb4863-f39d-41db-bf3c-1b04a45ac05b kind: Event lastTimestamp: "2026-09-18T16:49:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:52Z" name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t.18d6785149dffee6 namespace: kserve-ci-e2e-test resourceVersion: "27280" uid: ea85b244-dac7-40b3-962e-29846cab542d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T16:49:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 1fdb4863-f39d-41db-bf3c-1b04a45ac05b kind: Event lastTimestamp: "2026-09-18T16:59:19Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:49:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:59:19Z" name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t.18d67851a3052a26 namespace: kserve-ci-e2e-test resourceVersion: "31465" uid: d6bd3272-94a8-4ad5-bdd5-01c63727c69c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:59:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 1fdb4863-f39d-41db-bf3c-1b04a45ac05b kind: Event lastTimestamp: "2026-09-18T16:59:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:59:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:59:16Z" name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t.18d678d48d931c07 namespace: kserve-ci-e2e-test resourceVersion: "31431" uid: 635d04fb-4f6a-4324-b614-8fb743cf1f9e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:59:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 1fdb4863-f39d-41db-bf3c-1b04a45ac05b kind: Event lastTimestamp: "2026-09-18T16:59:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:59:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:59:16Z" name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t.18d678d48d93ab01 namespace: kserve-ci-e2e-test resourceVersion: "31432" uid: e3491d2d-5b55-428b-a176-9aecc0d828d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:59:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 1fdb4863-f39d-41db-bf3c-1b04a45ac05b kind: Event lastTimestamp: "2026-09-18T16:59:19Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:59:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:59:19Z" name: error-404-isvc-bb388-predictor-6d44f64874-nwx7t.18d678d56b7e3428 namespace: kserve-ci-e2e-test resourceVersion: "31464" uid: c911e671-e665-41da-8247-c1d55671451a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bb388-predictor-6d44f64874 namespace: kserve-ci-e2e-test resourceVersion: "27225" uid: ec810ff2-7a12-437d-9a41-ffbcf3d90c97 kind: Event lastTimestamp: "2026-09-18T16:49:51Z" message: 'Created pod: error-404-isvc-bb388-predictor-6d44f64874-nwx7t' metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:51Z" name: error-404-isvc-bb388-predictor-6d44f64874.18d67850f9ce0e81 namespace: kserve-ci-e2e-test resourceVersion: "27232" uid: b7230c8b-45d1-42cb-9e7e-1392f8290668 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bb388-predictor namespace: kserve-ci-e2e-test resourceVersion: "27223" uid: e991fd76-a180-41f9-a645-c79a10c49427 kind: Event lastTimestamp: "2026-09-18T16:49:51Z" message: Scaled up replica set error-404-isvc-bb388-predictor-6d44f64874 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:51Z" name: error-404-isvc-bb388-predictor.18d67850f8140532 namespace: kserve-ci-e2e-test resourceVersion: "27226" uid: 6d83504f-650f-49d3-8a36-17ab2c4a42fd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:50:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bb388-predictor namespace: kserve-ci-e2e-test resourceVersion: "27224" uid: 4ac36da4-e30c-4c81-812b-8ace901b3ea5 kind: Event lastTimestamp: "2026-09-18T16:50:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:50:36Z" name: error-404-isvc-bb388-predictor.18d6785476d5d23b namespace: kserve-ci-e2e-test resourceVersion: "27709" uid: 32dfd3d2-2451-410b-88ae-09d518660f82 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:50:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bb388-predictor namespace: kserve-ci-e2e-test resourceVersion: "27224" uid: 4ac36da4-e30c-4c81-812b-8ace901b3ea5 kind: Event lastTimestamp: "2026-09-18T16:50:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:50:36Z" name: error-404-isvc-bb388-predictor.18d67854773999db namespace: kserve-ci-e2e-test resourceVersion: "27710" uid: 6075e519-7cdb-4de8-98e3-d77c91249222 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:50:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bb388-predictor namespace: kserve-ci-e2e-test resourceVersion: "27414" uid: 4ac36da4-e30c-4c81-812b-8ace901b3ea5 kind: Event lastTimestamp: "2026-09-18T16:51:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:51:21Z" name: error-404-isvc-bb388-predictor.18d67857f5e0c2e1 namespace: kserve-ci-e2e-test resourceVersion: "28096" uid: 63c2be51-2cb2-45aa-8291-03167a14d93f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:50:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bb388-predictor namespace: kserve-ci-e2e-test resourceVersion: "27414" uid: 4ac36da4-e30c-4c81-812b-8ace901b3ea5 kind: Event lastTimestamp: "2026-09-18T16:51:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:51:21Z" name: error-404-isvc-bb388-predictor.18d67857f65f5a2b namespace: kserve-ci-e2e-test resourceVersion: "28098" uid: 4d616a74-80dc-463c-9f8b-8ec91c8083ad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:49:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bb388 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 325797cd-bbe4-469d-b4fb-68c39ba64ca5 kind: Event lastTimestamp: "2026-09-18T16:50:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:50:50Z" name: error-404-isvc-bb388.18d67850f81a01df namespace: kserve-ci-e2e-test resourceVersion: "27829" uid: af496fa9-2792-40f9-8d99-eb8df1527c9d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:50:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bb388 namespace: kserve-ci-e2e-test resourceVersion: "27826" uid: 325797cd-bbe4-469d-b4fb-68c39ba64ca5 kind: Event lastTimestamp: "2026-09-18T16:50:50Z" message: InferenceService [error-404-isvc-bb388] is Ready metadata: creationTimestamp: "2026-09-18T16:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:50:50Z" name: error-404-isvc-bb388.18d6785eb2c70907 namespace: kserve-ci-e2e-test resourceVersion: "27827" uid: 23069464-06fa-410d-bf16-438303ab4022 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:39:04.545362Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: fb03191d-d415-46ec-9ae0-bd9af3340e6c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk to ip-10-0-143-79.ec2.internal metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:39:04Z" name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk.18d677ba6f8b4729 namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: b2f9809c-ed5b-438a-b097-b6273b5ca266 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: fb03191d-d415-46ec-9ae0-bd9af3340e6c kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-de495-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:04Z" name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk.18d677ba7bb75b79 namespace: kserve-ci-e2e-test resourceVersion: "21543" uid: 2ca4b1c5-5151-4747-9d87-7a5986d945cc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk namespace: kserve-ci-e2e-test resourceVersion: "21536" uid: fb03191d-d415-46ec-9ae0-bd9af3340e6c kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:39:05Z" name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk.18d677bab5142e79 namespace: kserve-ci-e2e-test resourceVersion: "21565" uid: d5b1d251-2c43-4981-9e27-0a9671cbb213 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: fb03191d-d415-46ec-9ae0-bd9af3340e6c kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:05Z" name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk.18d677bab67e1ba4 namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: 51038615-7e1e-4279-a1f5-8813153627f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: fb03191d-d415-46ec-9ae0-bd9af3340e6c kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:05Z" name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk.18d677baba76428e namespace: kserve-ci-e2e-test resourceVersion: "21570" uid: 254932f8-d3e9-4a0d-aaa0-0eab26bf95e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: fb03191d-d415-46ec-9ae0-bd9af3340e6c kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:05Z" name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk.18d677babb13ee8e namespace: kserve-ci-e2e-test resourceVersion: "21571" uid: f037d236-22f7-4e22-93a3-54da373eed9e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: fb03191d-d415-46ec-9ae0-bd9af3340e6c kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:05Z" name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk.18d677babb1cda95 namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: bf081404-5047-4a49-8d33-b9934122494b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: fb03191d-d415-46ec-9ae0-bd9af3340e6c kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:05Z" name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk.18d677babe6c8ca6 namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: ad964cc7-eee9-4292-8e66-23a00bebdf35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: fb03191d-d415-46ec-9ae0-bd9af3340e6c kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:05Z" name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk.18d677babed951b1 namespace: kserve-ci-e2e-test resourceVersion: "21575" uid: 36f39ca2-626e-496b-b829-e02b8942a06a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:39:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: fb03191d-d415-46ec-9ae0-bd9af3340e6c kind: Event lastTimestamp: "2026-09-18T16:48:32Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:32Z" name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk.18d677bad4bad636 namespace: kserve-ci-e2e-test resourceVersion: "26316" uid: c09a1c58-50ff-4d58-8d43-28a036aaff62 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: fb03191d-d415-46ec-9ae0-bd9af3340e6c kind: Event lastTimestamp: "2026-09-18T16:48:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:29Z" name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk.18d6783e081f72c2 namespace: kserve-ci-e2e-test resourceVersion: "26246" uid: 583f8939-6b88-445f-ba44-ff4091be4a52 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: fb03191d-d415-46ec-9ae0-bd9af3340e6c kind: Event lastTimestamp: "2026-09-18T16:48:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:29Z" name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk.18d6783e081fd400 namespace: kserve-ci-e2e-test resourceVersion: "26249" uid: f29f5192-eff7-4298-86bf-3c381926b574 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: fb03191d-d415-46ec-9ae0-bd9af3340e6c kind: Event lastTimestamp: "2026-09-18T16:48:32Z" message: 'Readiness probe failed: Get "https://10.132.0.30:8643/healthz": dial tcp 10.132.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:32Z" name: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk.18d6783e9d275673 namespace: kserve-ci-e2e-test resourceVersion: "26314" uid: d510c438-dbbb-4aa6-a220-f7cb1b5e8733 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-79.ec2.internal source: component: kubelet host: ip-10-0-143-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-de495-predictor-7fb5ffb98b namespace: kserve-ci-e2e-test resourceVersion: "21517" uid: 954c991c-1618-47cf-b582-ed1d55f2dd9d kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: 'Created pod: error-404-isvc-de495-predictor-7fb5ffb98b-ztgqk' metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:04Z" name: error-404-isvc-de495-predictor-7fb5ffb98b.18d677ba6f1ebc1f namespace: kserve-ci-e2e-test resourceVersion: "21525" uid: b1a03a54-d907-49ae-8abb-4c5fd33bcd30 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-de495-predictor namespace: kserve-ci-e2e-test resourceVersion: "21515" uid: b4ea2240-ba2e-466b-9a41-13bbc4f2e782 kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: Scaled up replica set error-404-isvc-de495-predictor-7fb5ffb98b from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:04Z" name: error-404-isvc-de495-predictor.18d677ba6d43ba69 namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: ec8a7542-b952-4193-90fb-7db3aa784019 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:39:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de495-predictor namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: 6ba9164b-5e4f-4f0e-9f7a-78bbedae43cb kind: Event lastTimestamp: "2026-09-18T16:39:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:34Z" name: error-404-isvc-de495-predictor.18d677bdebced80f namespace: kserve-ci-e2e-test resourceVersion: "21837" uid: 81a39825-1a3c-4b12-bc22-cb66d6349bdc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:39:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de495-predictor namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: 6ba9164b-5e4f-4f0e-9f7a-78bbedae43cb kind: Event lastTimestamp: "2026-09-18T16:39:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:34Z" name: error-404-isvc-de495-predictor.18d677bdec3a15cf namespace: kserve-ci-e2e-test resourceVersion: "21838" uid: 14280438-095f-4d51-bdf5-74ae4f7f5fca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:39:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de495-predictor namespace: kserve-ci-e2e-test resourceVersion: "21709" uid: 6ba9164b-5e4f-4f0e-9f7a-78bbedae43cb kind: Event lastTimestamp: "2026-09-18T16:40:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:19Z" name: error-404-isvc-de495-predictor.18d677c4e9a32d10 namespace: kserve-ci-e2e-test resourceVersion: "22426" uid: bf527c05-e1db-4511-8c89-35ff2443c588 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:39:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-de495-predictor namespace: kserve-ci-e2e-test resourceVersion: "21709" uid: 6ba9164b-5e4f-4f0e-9f7a-78bbedae43cb kind: Event lastTimestamp: "2026-09-18T16:40:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:19Z" name: error-404-isvc-de495-predictor.18d677c4ea0b1048 namespace: kserve-ci-e2e-test resourceVersion: "22427" uid: 29f46315-04c3-4dd0-b4e4-b761d4738eda reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de495 namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: a90fc330-b394-481a-8eb2-5ceddf077b0b kind: Event lastTimestamp: "2026-09-18T16:39:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:52Z" name: error-404-isvc-de495.18d677ba6d47b076 namespace: kserve-ci-e2e-test resourceVersion: "22163" uid: d6b69ac3-ec8d-4f79-89f7-6da31d210d91 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de495 namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: a90fc330-b394-481a-8eb2-5ceddf077b0b kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-de495": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-de495": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:04Z" name: error-404-isvc-de495.18d677ba6dabd1b8 namespace: kserve-ci-e2e-test resourceVersion: "21521" uid: 70e8b109-45b0-4bed-af96-6ea8d7757d9d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de495 namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: a90fc330-b394-481a-8eb2-5ceddf077b0b kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-de495": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:04Z" name: error-404-isvc-de495.18d677ba6dac90fe namespace: kserve-ci-e2e-test resourceVersion: "21523" uid: beaf52ab-4cea-4a28-a1bd-41817d69cfe3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-de495 namespace: kserve-ci-e2e-test resourceVersion: "22161" uid: a90fc330-b394-481a-8eb2-5ceddf077b0b kind: Event lastTimestamp: "2026-09-18T16:39:52Z" message: InferenceService [error-404-isvc-de495] is Ready metadata: creationTimestamp: "2026-09-18T16:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:52Z" name: error-404-isvc-de495.18d677c593244bf5 namespace: kserve-ci-e2e-test resourceVersion: "22162" uid: 9b71f121-070f-4d7f-9092-ceec161f7d5f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:26:37.243862Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl to ip-10-0-135-110.ec2.internal metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:26:37Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d6770c70e76431 namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: 12ca1b78-da38-41bf-9325-1e5bad36bcd6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:26:37Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d6770c932e73b8 namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 48379e37-3621-4462-8523-d2dcafbfef0e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:37Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d6770c94c49bc5 namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: 85182565-c3d5-438b-acd9-9c5424387335 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:37Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d6770c984ae2a9 namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 3e89589e-394d-4803-88b8-251c0519c48e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:37Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d6770c98c14684 namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: fa8edcac-1003-43cd-9e2c-5f4dba89adb0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:26:43Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" metadata: creationTimestamp: "2026-09-18T16:26:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:43Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d6770e01e52711 namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: 4789eae7-c08f-4563-952a-cc908861dc22 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:26:55Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" in 11.117s (11.117s including waiting). Image size: 1607081368 bytes.' metadata: creationTimestamp: "2026-09-18T16:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:55Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d6771098879f09 namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: f89fb977-7531-408a-b86a-3237d43224cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:26:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:55Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d677109d7317a6 namespace: kserve-ci-e2e-test resourceVersion: "15214" uid: ff833a10-a907-4ecd-8428-3848a9268dd7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:26:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:55Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d677109e0e2939 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 1510183f-97e1-430a-bd1f-a02aeead2acf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:26:55Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-18T16:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:55Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d677109e201069 namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: bc5acb1f-4b36-48eb-9c0c-2d5a5cdee5bd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:26:59Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.784s (4.784s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-18T16:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:59Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d67711bb4f02e8 namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: a797ddea-5c54-410b-a957-007242d39706 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:27:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:27:00Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d67711bf90c360 namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: 302451da-df06-46e3-bd82-50925e1f6dbf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:27:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:27:00Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d67711c00b6ab0 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: d5ddc28a-14a6-48b3-9d27-f6db864fcb66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-18T16:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:29:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:29:03Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d67711ffe023a3 namespace: kserve-ci-e2e-test resourceVersion: "16706" uid: d63d49d7-c9a9-4318-a99c-90685753bd1b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:28:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:58Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d6772d6edd6dd1 namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: 51b1d685-42ee-44af-b1f0-01654b18f5d7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:28:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:58Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d6772d6eddd70d namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: 61d0ecef-bffa-464f-8ab0-360be7088f15 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a3f067c6-43a9-48de-9acb-7c1fcc44389e kind: Event lastTimestamp: "2026-09-18T16:29:02Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:29:02Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl.18d6772e2c50b9a0 namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: acc66e0a-a494-47d9-8b24-18cbfb2262f9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-66bf4f5cc4 namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: a1b1dc55-8a5b-4330-91b4-7b072ae3301a kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-66bf4f5cc4-lj7dl' metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:26:37Z" name: isvc-sklearn-graph-1-predictor-66bf4f5cc4.18d6770c706da029 namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: c5c2e936-37cd-47dc-99dd-543c36ae0c08 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14859" uid: aa0d737a-b1ff-47ab-bae4-d25c00a7388b kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-66bf4f5cc4 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:26:37Z" name: isvc-sklearn-graph-1-predictor.18d6770c68d92fa6 namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: d64067df-3a50-49ac-8514-6f833c12e516 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 3913deb7-a1d4-4139-9b1a-b7acb95b8000 kind: Event lastTimestamp: "2026-09-18T16:27:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:27:37Z" name: isvc-sklearn-graph-1-predictor.18d6770fee19d834 namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: fb398c5e-52f3-44a9-8471-45996df64c5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 3913deb7-a1d4-4139-9b1a-b7acb95b8000 kind: Event lastTimestamp: "2026-09-18T16:27:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:27:37Z" name: isvc-sklearn-graph-1-predictor.18d6770fee80c227 namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: 60c0a028-6706-47ef-b663-87bb23c7cdba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:27:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: 3913deb7-a1d4-4139-9b1a-b7acb95b8000 kind: Event lastTimestamp: "2026-09-18T16:28:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:52Z" name: isvc-sklearn-graph-1-predictor.18d67716ec8ed214 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 67bc7642-582a-4157-a2bc-07186e4b9faf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:27:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: 3913deb7-a1d4-4139-9b1a-b7acb95b8000 kind: Event lastTimestamp: "2026-09-18T16:28:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:52Z" name: isvc-sklearn-graph-1-predictor.18d67716ecf5679c namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: bf83adee-b389-44f4-89bc-99f7949c536a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: 09fc96a6-8a30-45e4-ac20-e802a55aa2de kind: Event lastTimestamp: "2026-09-18T16:28:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:28:23Z" name: isvc-sklearn-graph-1.18d6770c6ec4af66 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: efd28e76-5f66-4db7-b9e6-58977bc32d07 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16169" uid: 09fc96a6-8a30-45e4-ac20-e802a55aa2de kind: Event lastTimestamp: "2026-09-18T16:28:23Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-09-18T16:28:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:28:23Z" name: isvc-sklearn-graph-1.18d677251ca39f09 namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: b5c3b62b-a833-48bb-b11c-b299051e4450 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:26:37.739301Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk to ip-10-0-135-110.ec2.internal metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:26:37Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d6770c8e6f17e6 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 0a11008b-8948-4991-ae57-682ef996a89c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:37Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d6770c989fe6c3 namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: f370caad-4eb3-47b0-822c-9ad619e26380 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:26:38Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:26:38Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d6770cd437ba46 namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 8650d9b3-e3ed-4257-b8c3-f314c3d7ca75 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:26:38Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:38Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d6770cd59cd1a6 namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: 58d57693-eef2-4dbf-86a4-a62ed6ae0117 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:26:38Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:38Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d6770cd897769d namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: 89d45104-4378-4c8d-afea-6a21b0afc6ba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:26:38Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:38Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d6770cd90c8293 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 86a071bd-8d75-4d69-95db-9832e98b123f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:26:43Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" metadata: creationTimestamp: "2026-09-18T16:26:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:43Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d6770e01c99496 namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 126a6fb9-7ca8-4a10-a5ce-c29a0dca8471 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:26:55Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-2013" in 11.109s (11.109s including waiting). Image size: 1607081368 bytes.' metadata: creationTimestamp: "2026-09-18T16:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:55Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d6771097eff6d1 namespace: kserve-ci-e2e-test resourceVersion: "15211" uid: 0d1b4006-9fad-4ce2-9040-860a468e1df8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:26:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:55Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d677109d941f24 namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: 5fd49caa-57df-433e-9feb-d00a8197d8f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:26:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:55Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d677109e3faf1e namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: fa33c8b7-d3e7-46f8-bd12-ccf52a6f3052 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:26:55Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-18T16:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:55Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d677109e52206e namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: d2786500-97de-4c88-88f1-7fb788d20ff7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:26:59Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.78s (4.78s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-18T16:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:59Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d67711bb4534e0 namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: c4b012c3-ff7b-4b90-b201-08895bfe10c5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:27:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:27:00Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d67711bf828529 namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 115f2e0c-b71a-4500-ba3b-837655fb5dbb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:27:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:27:00Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d67711bffc8a8d namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 7c3492de-5df4-45ee-a9c8-b9cfe79d81eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-09-18T16:27:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:59Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d67711c3e401f0 namespace: kserve-ci-e2e-test resourceVersion: "16620" uid: f7e8cc93-7f92-4a40-be5d-3e3dbe811593 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:28:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:58Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d6772d6c7e0cc3 namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: 1d791877-40e5-46a6-816c-841e7e98892d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:28:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:58Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d6772d6c7e80ce namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: 9d3fab6c-a5ad-4a92-a89e-3cce72561b13 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 7dd084c0-f257-44bd-8197-18bc1aa1b0d9 kind: Event lastTimestamp: "2026-09-18T16:29:01Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:29:01Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk.18d6772df066cf43 namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: 73e0a2f2-b0f1-41d3-8bde-37edf61931d3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-b7ccc5f99 namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: 3fe3d517-4f20-4346-a47a-71f6a6ba6413 kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-b7ccc5f99-hx7fk' metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:26:37Z" name: isvc-sklearn-graph-2-predictor-b7ccc5f99.18d6770c8df571e9 namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: a63fb394-e631-4dc7-bfdd-bcdf961db050 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: 6c9a3b1b-4ffa-43f1-8153-70a5536953c2 kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-b7ccc5f99 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:26:37Z" name: isvc-sklearn-graph-2-predictor.18d6770c8af6201e namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 8721227f-f300-42bc-9109-f87d6a6a424a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: 7515e5b3-d61c-4713-8eee-949108aa75ee kind: Event lastTimestamp: "2026-09-18T16:27:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:27:37Z" name: isvc-sklearn-graph-2-predictor.18d6771009744024 namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: 77d48a75-af5c-43b6-bfc9-096c8c3f2d07 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: 7515e5b3-d61c-4713-8eee-949108aa75ee kind: Event lastTimestamp: "2026-09-18T16:27:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:27:37Z" name: isvc-sklearn-graph-2-predictor.18d6771009e08fae namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: f6412547-9f52-4be9-bbee-23a5969aae6f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:27:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: 7515e5b3-d61c-4713-8eee-949108aa75ee kind: Event lastTimestamp: "2026-09-18T16:28:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:52Z" name: isvc-sklearn-graph-2-predictor.18d6771707459ba2 namespace: kserve-ci-e2e-test resourceVersion: "16463" uid: 4409b0c8-4669-4cd0-8c84-8681bbb898fa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:27:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: 7515e5b3-d61c-4713-8eee-949108aa75ee kind: Event lastTimestamp: "2026-09-18T16:28:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:52Z" name: isvc-sklearn-graph-2-predictor.18d6771707a8ec46 namespace: kserve-ci-e2e-test resourceVersion: "16464" uid: b1cfe28b-0b95-458d-9eb9-93ee40755033 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14964" uid: 9bea99e5-9361-45f7-9fb0-51d5a6ffa675 kind: Event lastTimestamp: "2026-09-18T16:28:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:28:29Z" name: isvc-sklearn-graph-2.18d6770c8af62851 namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: 9d458033-9a52-46e6-8295-3060e6229ff9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 9bea99e5-9361-45f7-9fb0-51d5a6ffa675 kind: Event lastTimestamp: "2026-09-18T16:28:29Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-09-18T16:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:28:29Z" name: isvc-sklearn-graph-2.18d6772681c81fb8 namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: f647e606-e718-44e5-a0c5-8a3771df4ac1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:26:37.567608Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-jp7sd to ip-10-0-135-110.ec2.internal metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:26:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d6770c8433454c namespace: kserve-ci-e2e-test resourceVersion: "14956" uid: 0d5014f5-0e04-40b2-8496-446f591e50c5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:26:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d6770c929a216a namespace: kserve-ci-e2e-test resourceVersion: "15026" uid: a350a1e4-08e4-4646-8ffc-f710e2db6b6a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:26:39Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:26:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d6770d02439eb8 namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: 6551d3fc-b437-4911-a065-be0d915dad2a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:26:39Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2cbd0ba61f740962b2bad1a0e55e41574489d7afdb86fa314f3953b8c8af773e" already present on machine metadata: creationTimestamp: "2026-09-18T16:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d6770d04b4e7b3 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: db69aa2e-c639-419d-9898-b353a8b2b606 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:26:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-18T16:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d6770d0a04d638 namespace: kserve-ci-e2e-test resourceVersion: "15050" uid: 4209c468-71d0-4647-af9b-d419a9c3c905 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:26:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-18T16:26:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d6770d0b468f64 namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 16820e25-2081-44c3-8067-708e25a053ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:26:47Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-18T16:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:47Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d6770ef1d33b9a namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: 11f53f1f-371b-49fd-a5b3-1f61fff31e48 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:27:17Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 29.029s (29.029s including waiting). Image size: 1436472580 bytes.' metadata: creationTimestamp: "2026-09-18T16:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:27:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d67715b421d13b namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: a01d3877-1e0a-4758-8b9d-edc49cbf4d66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:27:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:27:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d67715b96a42b3 namespace: kserve-ci-e2e-test resourceVersion: "15416" uid: df763c23-4d98-46f7-8d23-67186e3b9f87 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:27:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:27:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d67715baaaf789 namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 9ec26514-4f82-4c92-bbe7-62749c03651f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:27:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:27:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d67715bac0c92c namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: f5ab2a04-33da-4e30-b4e3-d95b37924ad2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:27:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:27:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d67715c044d202 namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: bf1b7708-9bcd-450d-aa61-fb50bbaf735f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:27:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:27:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d67715c197bfa0 namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: 3eb2e1b4-ca6b-4f2a-a770-9937f7a35a25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:27:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:29:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:27:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:29:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d67715fa89cc62 namespace: kserve-ci-e2e-test resourceVersion: "16722" uid: 38852096-db41-41e0-94fa-6cb63cbe2bd5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:28:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:58Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d6772d70e21cb3 namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: ca7bd18d-c19b-4590-874c-2e359d14c71c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:28:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:58Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d6772d70e3453a namespace: kserve-ci-e2e-test resourceVersion: "16590" uid: 3d00d417-8272-4ac0-8414-7f1218a270d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 656d406f-8dff-43e6-95f5-4908cfcc3179 kind: Event lastTimestamp: "2026-09-18T16:29:04Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:29:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-jp7sd.18d6772d7ee7b72c namespace: kserve-ci-e2e-test resourceVersion: "16721" uid: 922520c1-cd79-42b0-9076-6a252cb42f29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: 090e74ce-dda9-4873-b30a-6f389d2c937b kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-jp7sd' metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:26:37Z" name: isvc-xgboost-graph-predictor-555fc69d56.18d6770c83b73e78 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: f91e9ff8-23ca-43b1-ba2d-3726887564ab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: b25c0c49-49c1-4a1c-8c75-e3d585794fc4 kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:26:37Z" name: isvc-xgboost-graph-predictor.18d6770c808a76bd namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 5fc9a722-bd76-4ad7-b3bb-6f2d530a2d35 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: 806cc0c1-a413-4aa8-9ea2-88b97c0c3d07 kind: Event lastTimestamp: "2026-09-18T16:27:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:27:52Z" name: isvc-xgboost-graph-predictor.18d6770fff181d69 namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: c5bc77fe-6a22-464c-b813-db62dc2f3d86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: 806cc0c1-a413-4aa8-9ea2-88b97c0c3d07 kind: Event lastTimestamp: "2026-09-18T16:27:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:27:52Z" name: isvc-xgboost-graph-predictor.18d6770fff85f5b2 namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 960e8e6f-92cd-4ffc-afda-8d0cd6afb7c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:28:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 806cc0c1-a413-4aa8-9ea2-88b97c0c3d07 kind: Event lastTimestamp: "2026-09-18T16:28:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:52Z" name: isvc-xgboost-graph-predictor.18d6772178e34a2f namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: e67de251-b811-420d-85ed-a3d76756130f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:28:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 806cc0c1-a413-4aa8-9ea2-88b97c0c3d07 kind: Event lastTimestamp: "2026-09-18T16:28:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:52Z" name: isvc-xgboost-graph-predictor.18d677217946a825 namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: 470e08b2-e62d-4a1d-b1c5-cd7190b9ac49 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: f80e9a7d-de3f-4a8f-9261-dc5f5487cc49 kind: Event lastTimestamp: "2026-09-18T16:28:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:28:34Z" name: isvc-xgboost-graph.18d6770c809a323b namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: fb942fb9-98e8-4498-8bba-4cde6ef3ca93 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: f80e9a7d-de3f-4a8f-9261-dc5f5487cc49 kind: Event lastTimestamp: "2026-09-18T16:28:34Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-09-18T16:28:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:28:34Z" name: isvc-xgboost-graph.18d67727b27f7264 namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: d2005385-f282-4401-9b15-07fdbc62c8c5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:28:47.212813Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7bcc66cdd8-fd944 namespace: kserve-ci-e2e-test resourceVersion: "16391" uid: 0561d2f9-4ade-479c-a95a-d5e1cd746f2f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-7bcc66cdd8-fd944 to ip-10-0-128-224.ec2.internal metadata: creationTimestamp: "2026-09-18T16:28:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:28:47Z" name: model-chainer-7bcc66cdd8-fd944.18d6772ab3a80f01 namespace: kserve-ci-e2e-test resourceVersion: "16396" uid: f2034d02-0a1b-487b-8262-ecd910b08d7f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:47Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7bcc66cdd8-fd944 namespace: kserve-ci-e2e-test resourceVersion: "16400" uid: 0561d2f9-4ade-479c-a95a-d5e1cd746f2f kind: Event lastTimestamp: "2026-09-18T16:28:47Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:28:47Z" name: model-chainer-7bcc66cdd8-fd944.18d6772ad5dcc170 namespace: kserve-ci-e2e-test resourceVersion: "16411" uid: 3749e2bd-b4d2-4394-9102-99a230aa940e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7bcc66cdd8-fd944 namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 0561d2f9-4ade-479c-a95a-d5e1cd746f2f kind: Event lastTimestamp: "2026-09-18T16:28:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" already present on machine metadata: creationTimestamp: "2026-09-18T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:47Z" name: model-chainer-7bcc66cdd8-fd944.18d6772ad741ede9 namespace: kserve-ci-e2e-test resourceVersion: "16413" uid: 4f3c7a45-8f5a-414a-acb3-a882c8a251a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7bcc66cdd8-fd944 namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 0561d2f9-4ade-479c-a95a-d5e1cd746f2f kind: Event lastTimestamp: "2026-09-18T16:28:47Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-09-18T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:47Z" name: model-chainer-7bcc66cdd8-fd944.18d6772ada13bf2f namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 966fd04a-3932-43c1-a1b7-bd3f629bd88d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7bcc66cdd8-fd944 namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 0561d2f9-4ade-479c-a95a-d5e1cd746f2f kind: Event lastTimestamp: "2026-09-18T16:28:47Z" message: Started container model-chainer metadata: creationTimestamp: "2026-09-18T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:47Z" name: model-chainer-7bcc66cdd8-fd944.18d6772ada80ce19 namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 9bf8fccf-ceb7-4444-8a51-e459ab33d95a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7bcc66cdd8-fd944 namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 0561d2f9-4ade-479c-a95a-d5e1cd746f2f kind: Event lastTimestamp: "2026-09-18T16:28:58Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-09-18T16:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:58Z" name: model-chainer-7bcc66cdd8-fd944.18d6772d660724f9 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: d0f4489d-f458-4e10-aaa4-689fdb85b6fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7bcc66cdd8-fd944 namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 0561d2f9-4ade-479c-a95a-d5e1cd746f2f kind: Event lastTimestamp: "2026-09-18T16:29:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:29:24Z" name: model-chainer-7bcc66cdd8-fd944.18d6772d76b3131e namespace: kserve-ci-e2e-test resourceVersion: "16906" uid: e09090ea-4d00-4c9f-9678-c6f11ded221e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-7bcc66cdd8 namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: ceff8287-0d77-4164-9bae-f4625c82742e kind: Event lastTimestamp: "2026-09-18T16:28:47Z" message: 'Created pod: model-chainer-7bcc66cdd8-fd944' metadata: creationTimestamp: "2026-09-18T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:47Z" name: model-chainer-7bcc66cdd8.18d6772ab2e7790a namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: 9bfba124-108b-44e7-9943-75119eeb4641 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: 242544d2-c78d-447c-a1a4-c9cf14adbc13 kind: Event lastTimestamp: "2026-09-18T16:28:47Z" message: Scaled up replica set model-chainer-7bcc66cdd8 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:47Z" name: model-chainer.18d6772ab069f9d5 namespace: kserve-ci-e2e-test resourceVersion: "16389" uid: 5da96234-7789-41c6-b910-25472789787a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16402" uid: 1560e64b-d1e7-4dd4-b7e2-eab4ddba0ce6 kind: Event lastTimestamp: "2026-09-18T16:28:47Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-09-18T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:28:47Z" name: model-chainer.18d6772aba4eaf7c namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: acd5dc93-e61f-475f-8146-bf9f2fcfda95 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:40:55.942668Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-630d7-56cf9d7d54-gs9ts namespace: kserve-ci-e2e-test resourceVersion: "22730" uid: eedb7ec0-db5e-44fc-a93f-7ccce6e43261 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-630d7-56cf9d7d54-gs9ts to ip-10-0-128-224.ec2.internal metadata: creationTimestamp: "2026-09-18T16:40:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:40:55Z" name: sequence-graph-630d7-56cf9d7d54-gs9ts.18d677d45f57417a namespace: kserve-ci-e2e-test resourceVersion: "22735" uid: 775f0870-84e8-429f-a60a-4bc534426b1d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:56Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-630d7-56cf9d7d54-gs9ts namespace: kserve-ci-e2e-test resourceVersion: "22733" uid: eedb7ec0-db5e-44fc-a93f-7ccce6e43261 kind: Event lastTimestamp: "2026-09-18T16:40:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-630d7-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:56Z" name: sequence-graph-630d7-56cf9d7d54-gs9ts.18d677d46ac5fafa namespace: kserve-ci-e2e-test resourceVersion: "22746" uid: 92780e26-f72b-4213-83a8-1f6e5af9cffe reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:57Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-630d7-56cf9d7d54-gs9ts namespace: kserve-ci-e2e-test resourceVersion: "22739" uid: eedb7ec0-db5e-44fc-a93f-7ccce6e43261 kind: Event lastTimestamp: "2026-09-18T16:40:57Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:40:57Z" name: sequence-graph-630d7-56cf9d7d54-gs9ts.18d677d4a50188d0 namespace: kserve-ci-e2e-test resourceVersion: "22755" uid: f384c536-bfe9-4a95-bc95-ace7a0725156 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-630d7} kind: Pod name: sequence-graph-630d7-56cf9d7d54-gs9ts namespace: kserve-ci-e2e-test resourceVersion: "22733" uid: eedb7ec0-db5e-44fc-a93f-7ccce6e43261 kind: Event lastTimestamp: "2026-09-18T16:40:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" already present on machine metadata: creationTimestamp: "2026-09-18T16:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:57Z" name: sequence-graph-630d7-56cf9d7d54-gs9ts.18d677d4a67276de namespace: kserve-ci-e2e-test resourceVersion: "22757" uid: ee108f71-f745-4f1d-ac4b-92fbe1fa5010 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-630d7} kind: Pod name: sequence-graph-630d7-56cf9d7d54-gs9ts namespace: kserve-ci-e2e-test resourceVersion: "22733" uid: eedb7ec0-db5e-44fc-a93f-7ccce6e43261 kind: Event lastTimestamp: "2026-09-18T16:40:57Z" message: 'Created container: sequence-graph-630d7' metadata: creationTimestamp: "2026-09-18T16:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:57Z" name: sequence-graph-630d7-56cf9d7d54-gs9ts.18d677d4a9caf59c namespace: kserve-ci-e2e-test resourceVersion: "22758" uid: a07119ee-6c06-48a6-b2bc-bb9191be40b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-630d7} kind: Pod name: sequence-graph-630d7-56cf9d7d54-gs9ts namespace: kserve-ci-e2e-test resourceVersion: "22733" uid: eedb7ec0-db5e-44fc-a93f-7ccce6e43261 kind: Event lastTimestamp: "2026-09-18T16:40:57Z" message: Started container sequence-graph-630d7 metadata: creationTimestamp: "2026-09-18T16:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:57Z" name: sequence-graph-630d7-56cf9d7d54-gs9ts.18d677d4aa446586 namespace: kserve-ci-e2e-test resourceVersion: "22759" uid: 1268c68e-ba38-45e4-b9ea-73545c1576e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-630d7} kind: Pod name: sequence-graph-630d7-56cf9d7d54-gs9ts namespace: kserve-ci-e2e-test resourceVersion: "22733" uid: eedb7ec0-db5e-44fc-a93f-7ccce6e43261 kind: Event lastTimestamp: "2026-09-18T16:49:10Z" message: Stopping container sequence-graph-630d7 metadata: creationTimestamp: "2026-09-18T16:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:10Z" name: sequence-graph-630d7-56cf9d7d54-gs9ts.18d6784796d5c88b namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: a2479b44-9cf4-4cd3-bfb5-daf1ce6b56b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-630d7} kind: Pod name: sequence-graph-630d7-56cf9d7d54-gs9ts namespace: kserve-ci-e2e-test resourceVersion: "22733" uid: eedb7ec0-db5e-44fc-a93f-7ccce6e43261 kind: Event lastTimestamp: "2026-09-18T16:49:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:39Z" name: sequence-graph-630d7-56cf9d7d54-gs9ts.18d678485c169ff4 namespace: kserve-ci-e2e-test resourceVersion: "27003" uid: ed0b0b09-841c-49b3-a292-3e7f7300bfb2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-630d7-56cf9d7d54 namespace: kserve-ci-e2e-test resourceVersion: "22727" uid: 508689c3-511e-470f-bd50-41a0573d91d0 kind: Event lastTimestamp: "2026-09-18T16:40:55Z" message: 'Created pod: sequence-graph-630d7-56cf9d7d54-gs9ts' metadata: creationTimestamp: "2026-09-18T16:40:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:55Z" name: sequence-graph-630d7-56cf9d7d54.18d677d45e921d60 namespace: kserve-ci-e2e-test resourceVersion: "22732" uid: 62b8a70a-d5a2-4fdc-a63c-8d4e0f5faf00 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-630d7 namespace: kserve-ci-e2e-test resourceVersion: "22725" uid: 9401c026-310b-4f8b-b6d0-0983c4b711e0 kind: Event lastTimestamp: "2026-09-18T16:40:55Z" message: Scaled up replica set sequence-graph-630d7-56cf9d7d54 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:40:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:55Z" name: sequence-graph-630d7.18d677d45dbae42f namespace: kserve-ci-e2e-test resourceVersion: "22728" uid: 98819a70-8412-4c34-a283-b7df80ccf06e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-630d7 namespace: kserve-ci-e2e-test resourceVersion: "22741" uid: 861e49d8-168e-42d9-b7d7-5cff839352ac kind: Event lastTimestamp: "2026-09-18T16:40:56Z" message: InferenceGraph [sequence-graph-630d7] is Ready metadata: creationTimestamp: "2026-09-18T16:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:40:56Z" name: sequence-graph-630d7.18d677d46304d206 namespace: kserve-ci-e2e-test resourceVersion: "22743" uid: 5df14e41-caa5-46ee-b9f5-95963d564c1f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:41:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-630d7 namespace: kserve-ci-e2e-test resourceVersion: "22726" uid: 0906fdd4-3279-4d24-b5a9-43b4cd303943 kind: Event lastTimestamp: "2026-09-18T16:41:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:41:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:41:25Z" name: sequence-graph-630d7.18d677d7dc406cca namespace: kserve-ci-e2e-test resourceVersion: "22980" uid: b82c305b-ecb3-4d03-bc18-532eb93c886d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:41:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-630d7 namespace: kserve-ci-e2e-test resourceVersion: "22726" uid: 0906fdd4-3279-4d24-b5a9-43b4cd303943 kind: Event lastTimestamp: "2026-09-18T16:41:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:41:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:41:25Z" name: sequence-graph-630d7.18d677d7dca06669 namespace: kserve-ci-e2e-test resourceVersion: "22981" uid: ef86bba7-553a-4ef4-a2d6-3825a1dfc654 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:39:34.748504Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7c9ed-75c5c964bc-f49kw namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: 23a09dee-28ef-43ec-80ec-169441f825ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-7c9ed-75c5c964bc-f49kw to ip-10-0-128-224.ec2.internal metadata: creationTimestamp: "2026-09-18T16:39:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:39:34Z" name: sequence-graph-7c9ed-75c5c964bc-f49kw.18d677c177ca930e namespace: kserve-ci-e2e-test resourceVersion: "21857" uid: 63c2fce0-46f0-46bf-9dab-146cc661d977 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:35Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7c9ed-75c5c964bc-f49kw namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: 23a09dee-28ef-43ec-80ec-169441f825ff kind: Event lastTimestamp: "2026-09-18T16:39:35Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:39:35Z" name: sequence-graph-7c9ed-75c5c964bc-f49kw.18d677c199bc823b namespace: kserve-ci-e2e-test resourceVersion: "21875" uid: e069d687-e54c-4b20-bcbf-50f7326c132b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c9ed} kind: Pod name: sequence-graph-7c9ed-75c5c964bc-f49kw namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 23a09dee-28ef-43ec-80ec-169441f825ff kind: Event lastTimestamp: "2026-09-18T16:39:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:35Z" name: sequence-graph-7c9ed-75c5c964bc-f49kw.18d677c19aeffd41 namespace: kserve-ci-e2e-test resourceVersion: "21877" uid: 6fda56ba-6099-49a1-9ada-b00c4a82a372 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c9ed} kind: Pod name: sequence-graph-7c9ed-75c5c964bc-f49kw namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 23a09dee-28ef-43ec-80ec-169441f825ff kind: Event lastTimestamp: "2026-09-18T16:39:35Z" message: 'Created container: sequence-graph-7c9ed' metadata: creationTimestamp: "2026-09-18T16:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:35Z" name: sequence-graph-7c9ed-75c5c964bc-f49kw.18d677c19dd6c2be namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: 1d7850fc-d3fe-42df-b7e9-79e93a6ee47c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c9ed} kind: Pod name: sequence-graph-7c9ed-75c5c964bc-f49kw namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 23a09dee-28ef-43ec-80ec-169441f825ff kind: Event lastTimestamp: "2026-09-18T16:39:35Z" message: Started container sequence-graph-7c9ed metadata: creationTimestamp: "2026-09-18T16:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:35Z" name: sequence-graph-7c9ed-75c5c964bc-f49kw.18d677c19e46e66d namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: 27bd6693-7f00-4edb-9c17-c468ac6cbefb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c9ed} kind: Pod name: sequence-graph-7c9ed-75c5c964bc-f49kw namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 23a09dee-28ef-43ec-80ec-169441f825ff kind: Event lastTimestamp: "2026-09-18T16:39:45Z" message: Stopping container sequence-graph-7c9ed metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:45Z" name: sequence-graph-7c9ed-75c5c964bc-f49kw.18d677c4038d4df8 namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: a37be558-3c87-4818-9bac-d3e54fa42ea7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c9ed} kind: Pod name: sequence-graph-7c9ed-75c5c964bc-f49kw namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 23a09dee-28ef-43ec-80ec-169441f825ff kind: Event lastTimestamp: "2026-09-18T16:40:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:40:11Z" name: sequence-graph-7c9ed-75c5c964bc-f49kw.18d677c443e1a0e1 namespace: kserve-ci-e2e-test resourceVersion: "22327" uid: 3106e0b4-fbda-4aac-9733-c2385f33d63f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-7c9ed-75c5c964bc namespace: kserve-ci-e2e-test resourceVersion: "21849" uid: 6f8bbca9-de4b-46a6-ac14-e858fc7a3e66 kind: Event lastTimestamp: "2026-09-18T16:39:34Z" message: 'Created pod: sequence-graph-7c9ed-75c5c964bc-f49kw' metadata: creationTimestamp: "2026-09-18T16:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:34Z" name: sequence-graph-7c9ed-75c5c964bc.18d677c17726bb3f namespace: kserve-ci-e2e-test resourceVersion: "21853" uid: 7f5999c1-dc25-4ba5-bf96-7132d268331f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-7c9ed namespace: kserve-ci-e2e-test resourceVersion: "21847" uid: 5c632809-6971-4396-a491-174bec25b5c2 kind: Event lastTimestamp: "2026-09-18T16:39:34Z" message: Scaled up replica set sequence-graph-7c9ed-75c5c964bc from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:34Z" name: sequence-graph-7c9ed.18d677c176526f90 namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 7ca84940-5aeb-4335-9a21-92b14e1a1a17 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-7c9ed namespace: kserve-ci-e2e-test resourceVersion: "21865" uid: bd14277f-09cd-424f-95d9-fac5cc1e2fec kind: Event lastTimestamp: "2026-09-18T16:39:34Z" message: InferenceGraph [sequence-graph-7c9ed] is Ready metadata: creationTimestamp: "2026-09-18T16:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:34Z" name: sequence-graph-7c9ed.18d677c17bfac3fe namespace: kserve-ci-e2e-test resourceVersion: "21867" uid: 344d8520-d5b4-4bfd-a0d6-7f47669da875 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:30:09.015780Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8d780-567975d654-5cz79 namespace: kserve-ci-e2e-test resourceVersion: "17312" uid: 0ec122de-f489-46ae-8b0e-358868fbca71 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8d780-567975d654-5cz79 to ip-10-0-128-224.ec2.internal metadata: creationTimestamp: "2026-09-18T16:30:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:30:09Z" name: sequence-graph-8d780-567975d654-5cz79.18d6773dbf7e4fca namespace: kserve-ci-e2e-test resourceVersion: "17317" uid: 18e44ec4-044c-47cc-b473-f963594e7e36 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:30:09Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8d780-567975d654-5cz79 namespace: kserve-ci-e2e-test resourceVersion: "17321" uid: 0ec122de-f489-46ae-8b0e-358868fbca71 kind: Event lastTimestamp: "2026-09-18T16:30:09Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:30:09Z" name: sequence-graph-8d780-567975d654-5cz79.18d6773de170b9f3 namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 533757ed-57fd-4758-8f6e-d2fbdcf01137 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d780} kind: Pod name: sequence-graph-8d780-567975d654-5cz79 namespace: kserve-ci-e2e-test resourceVersion: "17313" uid: 0ec122de-f489-46ae-8b0e-358868fbca71 kind: Event lastTimestamp: "2026-09-18T16:30:09Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" already present on machine metadata: creationTimestamp: "2026-09-18T16:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:30:09Z" name: sequence-graph-8d780-567975d654-5cz79.18d6773de2ee4523 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: e0efa844-3910-4777-a23c-95880808f59e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d780} kind: Pod name: sequence-graph-8d780-567975d654-5cz79 namespace: kserve-ci-e2e-test resourceVersion: "17313" uid: 0ec122de-f489-46ae-8b0e-358868fbca71 kind: Event lastTimestamp: "2026-09-18T16:30:09Z" message: 'Created container: sequence-graph-8d780' metadata: creationTimestamp: "2026-09-18T16:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:30:09Z" name: sequence-graph-8d780-567975d654-5cz79.18d6773de5d6ce68 namespace: kserve-ci-e2e-test resourceVersion: "17337" uid: 4ab19af3-8694-4538-a013-06546630e0bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d780} kind: Pod name: sequence-graph-8d780-567975d654-5cz79 namespace: kserve-ci-e2e-test resourceVersion: "17313" uid: 0ec122de-f489-46ae-8b0e-358868fbca71 kind: Event lastTimestamp: "2026-09-18T16:30:09Z" message: Started container sequence-graph-8d780 metadata: creationTimestamp: "2026-09-18T16:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:30:09Z" name: sequence-graph-8d780-567975d654-5cz79.18d6773de64ccf0b namespace: kserve-ci-e2e-test resourceVersion: "17338" uid: cfaa6d86-90f6-4729-90f6-135134944c9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d780} kind: Pod name: sequence-graph-8d780-567975d654-5cz79 namespace: kserve-ci-e2e-test resourceVersion: "17313" uid: 0ec122de-f489-46ae-8b0e-358868fbca71 kind: Event lastTimestamp: "2026-09-18T16:38:24Z" message: Stopping container sequence-graph-8d780 metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:24Z" name: sequence-graph-8d780-567975d654-5cz79.18d677b11d840d29 namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: dbe486a3-25dd-4c58-a975-ca99a1481c17 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:38:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d780} kind: Pod name: sequence-graph-8d780-567975d654-5cz79 namespace: kserve-ci-e2e-test resourceVersion: "17313" uid: 0ec122de-f489-46ae-8b0e-358868fbca71 kind: Event lastTimestamp: "2026-09-18T16:38:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:51Z" name: sequence-graph-8d780-567975d654-5cz79.18d677b1901ed30e namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: 78e51ad0-afc6-47a8-ab7b-4d2b980eac55 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:30:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8d780-567975d654 namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: 14e7208a-8079-4ee3-ac7f-35b3fd82d6b8 kind: Event lastTimestamp: "2026-09-18T16:30:09Z" message: 'Created pod: sequence-graph-8d780-567975d654-5cz79' metadata: creationTimestamp: "2026-09-18T16:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:30:09Z" name: sequence-graph-8d780-567975d654.18d6773dbef0d3ab namespace: kserve-ci-e2e-test resourceVersion: "17314" uid: 6fad2803-d9bf-4707-a655-32e6d507921d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:30:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8d780 namespace: kserve-ci-e2e-test resourceVersion: "17307" uid: 614da1d7-f6ee-46f0-8475-95a7d58b1494 kind: Event lastTimestamp: "2026-09-18T16:30:08Z" message: Scaled up replica set sequence-graph-8d780-567975d654 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:30:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:30:08Z" name: sequence-graph-8d780.18d6773dbde228a4 namespace: kserve-ci-e2e-test resourceVersion: "17310" uid: 5cd84fb7-987d-40de-94da-359cb247d159 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:30:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8d780 namespace: kserve-ci-e2e-test resourceVersion: "17327" uid: 06d6654f-5f37-45b6-a1d2-faa7dbe69aef kind: Event lastTimestamp: "2026-09-18T16:30:09Z" message: InferenceGraph [sequence-graph-8d780] is Ready metadata: creationTimestamp: "2026-09-18T16:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:30:09Z" name: sequence-graph-8d780.18d6773dc4b5cfdc namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: 31007146-b0a3-42af-b8d8-7589fa5dd732 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:30:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8d780 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 7818b02a-63a1-429e-ab06-625db57e4d45 kind: Event lastTimestamp: "2026-09-18T16:30:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:30:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:30:23Z" name: sequence-graph-8d780.18d677413c60c8c2 namespace: kserve-ci-e2e-test resourceVersion: "17450" uid: 86a7d9b1-853b-4860-9264-11fe68d8cf10 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:30:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8d780 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 7818b02a-63a1-429e-ab06-625db57e4d45 kind: Event lastTimestamp: "2026-09-18T16:30:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:30:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:30:24Z" name: sequence-graph-8d780.18d677413ce8d4d0 namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: a0f9d233-d756-4ae7-a08e-993a6668a3fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:30:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8d780 namespace: kserve-ci-e2e-test resourceVersion: "17449" uid: 7818b02a-63a1-429e-ab06-625db57e4d45 kind: Event lastTimestamp: "2026-09-18T16:30:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:30:39Z" name: sequence-graph-8d780.18d67744bbe6489a namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: 80151bde-6d1e-4b40-a761-48b88fdd6cd0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:30:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8d780 namespace: kserve-ci-e2e-test resourceVersion: "17449" uid: 7818b02a-63a1-429e-ab06-625db57e4d45 kind: Event lastTimestamp: "2026-09-18T16:30:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:30:39Z" name: sequence-graph-8d780.18d67744bc833f63 namespace: kserve-ci-e2e-test resourceVersion: "17557" uid: 5eec2630-86f0-4514-978b-9f672c77e164 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:49:39.748481Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b3de6-586d65fc7b-lczd2 namespace: kserve-ci-e2e-test resourceVersion: "27020" uid: 5b5b78e9-4e2e-4518-90bb-5fe642eda8ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-b3de6-586d65fc7b-lczd2 to ip-10-0-128-224.ec2.internal metadata: creationTimestamp: "2026-09-18T16:49:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:49:39Z" name: splitter-graph-b3de6-586d65fc7b-lczd2.18d6784e54999c14 namespace: kserve-ci-e2e-test resourceVersion: "27025" uid: fa994f88-2239-4647-b327-c8896ed657fc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:40Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b3de6-586d65fc7b-lczd2 namespace: kserve-ci-e2e-test resourceVersion: "27030" uid: 5b5b78e9-4e2e-4518-90bb-5fe642eda8ff kind: Event lastTimestamp: "2026-09-18T16:49:40Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:49:40Z" name: splitter-graph-b3de6-586d65fc7b-lczd2.18d6784e76680af1 namespace: kserve-ci-e2e-test resourceVersion: "27043" uid: 719f2646-6a48-42d1-a5bf-db789bf6a0d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b3de6} kind: Pod name: splitter-graph-b3de6-586d65fc7b-lczd2 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 5b5b78e9-4e2e-4518-90bb-5fe642eda8ff kind: Event lastTimestamp: "2026-09-18T16:49:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" already present on machine metadata: creationTimestamp: "2026-09-18T16:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:40Z" name: splitter-graph-b3de6-586d65fc7b-lczd2.18d6784e77a84fa8 namespace: kserve-ci-e2e-test resourceVersion: "27045" uid: bcade0cb-ad80-4c00-93a8-3b6fef71aaab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b3de6} kind: Pod name: splitter-graph-b3de6-586d65fc7b-lczd2 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 5b5b78e9-4e2e-4518-90bb-5fe642eda8ff kind: Event lastTimestamp: "2026-09-18T16:49:40Z" message: 'Created container: splitter-graph-b3de6' metadata: creationTimestamp: "2026-09-18T16:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:40Z" name: splitter-graph-b3de6-586d65fc7b-lczd2.18d6784e7aca5549 namespace: kserve-ci-e2e-test resourceVersion: "27047" uid: cf818791-b0a8-45d2-8596-573d908161b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b3de6} kind: Pod name: splitter-graph-b3de6-586d65fc7b-lczd2 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 5b5b78e9-4e2e-4518-90bb-5fe642eda8ff kind: Event lastTimestamp: "2026-09-18T16:49:40Z" message: Started container splitter-graph-b3de6 metadata: creationTimestamp: "2026-09-18T16:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:40Z" name: splitter-graph-b3de6-586d65fc7b-lczd2.18d6784e7b38515c namespace: kserve-ci-e2e-test resourceVersion: "27048" uid: 296269a4-404a-4887-822f-990fb54994da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b3de6} kind: Pod name: splitter-graph-b3de6-586d65fc7b-lczd2 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 5b5b78e9-4e2e-4518-90bb-5fe642eda8ff kind: Event lastTimestamp: "2026-09-18T16:49:50Z" message: Stopping container splitter-graph-b3de6 metadata: creationTimestamp: "2026-09-18T16:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:50Z" name: splitter-graph-b3de6-586d65fc7b-lczd2.18d67850e69e3aef namespace: kserve-ci-e2e-test resourceVersion: "27155" uid: 1bb22628-2217-4574-8d01-7bd61cd1f5ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b3de6} kind: Pod name: splitter-graph-b3de6-586d65fc7b-lczd2 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 5b5b78e9-4e2e-4518-90bb-5fe642eda8ff kind: Event lastTimestamp: "2026-09-18T16:50:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:50:17Z" name: splitter-graph-b3de6-586d65fc7b-lczd2.18d678513c4cc520 namespace: kserve-ci-e2e-test resourceVersion: "27511" uid: 172b2fc0-4219-4cb6-85bf-fd73d36e6da4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-b3de6-586d65fc7b namespace: kserve-ci-e2e-test resourceVersion: "27017" uid: 8bad75a0-987b-44df-ab35-f0a48177b1a4 kind: Event lastTimestamp: "2026-09-18T16:49:39Z" message: 'Created pod: splitter-graph-b3de6-586d65fc7b-lczd2' metadata: creationTimestamp: "2026-09-18T16:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:39Z" name: splitter-graph-b3de6-586d65fc7b.18d6784e53d24b04 namespace: kserve-ci-e2e-test resourceVersion: "27022" uid: 2cee8cce-c7a6-4aad-9bc6-8b2e50a3ef62 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-b3de6 namespace: kserve-ci-e2e-test resourceVersion: "27015" uid: d00f0927-5544-499b-849f-b43845fcff1f kind: Event lastTimestamp: "2026-09-18T16:49:39Z" message: Scaled up replica set splitter-graph-b3de6-586d65fc7b from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:39Z" name: splitter-graph-b3de6.18d6784e52fa8318 namespace: kserve-ci-e2e-test resourceVersion: "27018" uid: 92cbe617-8429-4005-a3f8-3cccc7e51ca5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-b3de6 namespace: kserve-ci-e2e-test resourceVersion: "27032" uid: dd618bf2-13e3-497f-996d-dec9e58b4c4a kind: Event lastTimestamp: "2026-09-18T16:49:39Z" message: InferenceGraph [splitter-graph-b3de6] is Ready metadata: creationTimestamp: "2026-09-18T16:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:49:39Z" name: splitter-graph-b3de6.18d6784e595816a2 namespace: kserve-ci-e2e-test resourceVersion: "27034" uid: 41586e20-069a-46b4-af81-4d3b1e940d55 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:51:01.025006Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-bb388-5b7597b4b8-nsmq4 namespace: kserve-ci-e2e-test resourceVersion: "27921" uid: 31a8d449-a3ee-4f1b-bd42-810e612dc445 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-bb388-5b7597b4b8-nsmq4 to ip-10-0-128-224.ec2.internal metadata: creationTimestamp: "2026-09-18T16:51:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:51:01Z" name: splitter-graph-bb388-5b7597b4b8-nsmq4.18d67861410ef8f6 namespace: kserve-ci-e2e-test resourceVersion: "27926" uid: 099d87a9-2174-4430-8b0e-49331be1c982 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:51:01Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-bb388-5b7597b4b8-nsmq4 namespace: kserve-ci-e2e-test resourceVersion: "27930" uid: 31a8d449-a3ee-4f1b-bd42-810e612dc445 kind: Event lastTimestamp: "2026-09-18T16:51:01Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:51:01Z" name: splitter-graph-bb388-5b7597b4b8-nsmq4.18d6786162e1f4ff namespace: kserve-ci-e2e-test resourceVersion: "27941" uid: 326c1cdb-f6ba-4cc7-8d8d-18ec31032d9e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:51:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bb388} kind: Pod name: splitter-graph-bb388-5b7597b4b8-nsmq4 namespace: kserve-ci-e2e-test resourceVersion: "27923" uid: 31a8d449-a3ee-4f1b-bd42-810e612dc445 kind: Event lastTimestamp: "2026-09-18T16:51:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" already present on machine metadata: creationTimestamp: "2026-09-18T16:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:51:01Z" name: splitter-graph-bb388-5b7597b4b8-nsmq4.18d67861642c1085 namespace: kserve-ci-e2e-test resourceVersion: "27944" uid: 23ee2535-6818-4bfe-9a22-5164bc1b7b04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:51:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bb388} kind: Pod name: splitter-graph-bb388-5b7597b4b8-nsmq4 namespace: kserve-ci-e2e-test resourceVersion: "27923" uid: 31a8d449-a3ee-4f1b-bd42-810e612dc445 kind: Event lastTimestamp: "2026-09-18T16:51:01Z" message: 'Created container: splitter-graph-bb388' metadata: creationTimestamp: "2026-09-18T16:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:51:01Z" name: splitter-graph-bb388-5b7597b4b8-nsmq4.18d678616758b818 namespace: kserve-ci-e2e-test resourceVersion: "27947" uid: 17faed08-e848-4a90-9ebb-ef4153bb2f16 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:51:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bb388} kind: Pod name: splitter-graph-bb388-5b7597b4b8-nsmq4 namespace: kserve-ci-e2e-test resourceVersion: "27923" uid: 31a8d449-a3ee-4f1b-bd42-810e612dc445 kind: Event lastTimestamp: "2026-09-18T16:51:01Z" message: Started container splitter-graph-bb388 metadata: creationTimestamp: "2026-09-18T16:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:51:01Z" name: splitter-graph-bb388-5b7597b4b8-nsmq4.18d6786167ca5a7c namespace: kserve-ci-e2e-test resourceVersion: "27948" uid: ca3bd2f7-ac5d-42da-9163-eead892a6ddd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:59:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bb388} kind: Pod name: splitter-graph-bb388-5b7597b4b8-nsmq4 namespace: kserve-ci-e2e-test resourceVersion: "27923" uid: 31a8d449-a3ee-4f1b-bd42-810e612dc445 kind: Event lastTimestamp: "2026-09-18T16:59:16Z" message: Stopping container splitter-graph-bb388 metadata: creationTimestamp: "2026-09-18T16:59:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:59:16Z" name: splitter-graph-bb388-5b7597b4b8-nsmq4.18d678d4842ac39d namespace: kserve-ci-e2e-test resourceVersion: "31400" uid: db026979-7940-4712-a5a0-9ceab3da8230 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:59:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bb388} kind: Pod name: splitter-graph-bb388-5b7597b4b8-nsmq4 namespace: kserve-ci-e2e-test resourceVersion: "27923" uid: 31a8d449-a3ee-4f1b-bd42-810e612dc445 kind: Event lastTimestamp: "2026-09-18T16:59:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:59:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:59:43Z" name: splitter-graph-bb388-5b7597b4b8-nsmq4.18d678d517d13e97 namespace: kserve-ci-e2e-test resourceVersion: "31639" uid: 82c10391-c0b8-434e-8f53-a12b414a15fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:51:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-bb388-5b7597b4b8 namespace: kserve-ci-e2e-test resourceVersion: "27918" uid: 6e631bb1-b37f-414b-a8e7-4b73bb83b954 kind: Event lastTimestamp: "2026-09-18T16:51:01Z" message: 'Created pod: splitter-graph-bb388-5b7597b4b8-nsmq4' metadata: creationTimestamp: "2026-09-18T16:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:51:01Z" name: splitter-graph-bb388-5b7597b4b8.18d67861406dcc50 namespace: kserve-ci-e2e-test resourceVersion: "27922" uid: 1fe4ac36-47b6-47b2-8661-439026c9369a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:51:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-bb388 namespace: kserve-ci-e2e-test resourceVersion: "27916" uid: 2281c360-c117-431d-a281-44ead7c8b97c kind: Event lastTimestamp: "2026-09-18T16:51:01Z" message: Scaled up replica set splitter-graph-bb388-5b7597b4b8 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:51:01Z" name: splitter-graph-bb388.18d678613f95d70e namespace: kserve-ci-e2e-test resourceVersion: "27919" uid: bf181059-089c-46b1-bfe8-f6742fd5d3b5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:51:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-bb388 namespace: kserve-ci-e2e-test resourceVersion: "27935" uid: 894b79ed-c1fb-4bda-8d9b-9c59fda3b6f2 kind: Event lastTimestamp: "2026-09-18T16:51:01Z" message: InferenceGraph [splitter-graph-bb388] is Ready metadata: creationTimestamp: "2026-09-18T16:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:51:01Z" name: splitter-graph-bb388.18d6786145185f43 namespace: kserve-ci-e2e-test resourceVersion: "27937" uid: f69626b6-d289-4d91-b890-cfb057e8ea5f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:51:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-bb388 namespace: kserve-ci-e2e-test resourceVersion: "27917" uid: 4d810f01-c0e6-49a0-a7a1-2359491fd487 kind: Event lastTimestamp: "2026-09-18T16:51:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:51:31Z" name: splitter-graph-bb388.18d67864be73e01f namespace: kserve-ci-e2e-test resourceVersion: "28165" uid: 28f3ef61-6e24-4805-bad2-97ee5503dd50 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:51:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-bb388 namespace: kserve-ci-e2e-test resourceVersion: "27917" uid: 4d810f01-c0e6-49a0-a7a1-2359491fd487 kind: Event lastTimestamp: "2026-09-18T16:51:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:51:31Z" name: splitter-graph-bb388.18d67864beded0ca namespace: kserve-ci-e2e-test resourceVersion: "28166" uid: 6f16c189-5e11-4300-91f9-7777d28efd87 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:26:37.344642Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14900" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 to ip-10-0-135-110.ec2.internal metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:26:37Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d6770c76e92061 namespace: kserve-ci-e2e-test resourceVersion: "14906" uid: 8c5afd12-72e2-40f7-b25b-92c4b1b46e07 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-191f9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:37Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d6770c869517ce namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: 30ce1189-c64c-4992-a18b-8ee80d4b932a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14911" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:26:38Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:26:38Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d6770cbcb0d057 namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: 525bb7b3-ec24-401e-aa31-052cecc3270c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:26:38Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-2013" metadata: creationTimestamp: "2026-09-18T16:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:38Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d6770cbe19acb4 namespace: kserve-ci-e2e-test resourceVersion: "15034" uid: 9ab8e92b-3c55-4ce1-96f1-71f66e25d593 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:26:55Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-2013" in 16.533s (16.533s including waiting). Image size: 1386037849 bytes.' metadata: creationTimestamp: "2026-09-18T16:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:55Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d6771097940e73 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: c88b62f4-efad-411e-8ca8-984e79af30ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:26:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:55Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d677109d835907 namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: a599c604-e98f-4f71-8639-6b63dfc5e1a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:26:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:55Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d677109e5732af namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: 29e7c89c-ecc6-47bd-ba6e-ddd4006a1f46 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:26:55Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-18T16:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:55Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d677109e65f980 namespace: kserve-ci-e2e-test resourceVersion: "15222" uid: 584d9ec0-cd9f-42e1-89f6-93388b4695cb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:26:59Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.731s (4.731s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-18T16:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:26:59Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d67711b868a6d9 namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: d9c8f84c-9528-49a0-a87a-0e00ce48503e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:27:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:27:00Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d67711bf4b3004 namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: ef07480a-6511-4927-b849-f7cf97a2ddde reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:27:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:27:00Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d67711bfc608aa namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 7218abbf-7e0b-4dd2-81db-3cfe4a38a84b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:27:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:27:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:27:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:27:46Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d67711c35651d3 namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: c102d239-d84a-4a48-bb98-367c26bbbb7d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:28:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:18Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d67723fff62dcd namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 60968372-4be9-44e9-b736-0452e3c38f4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:28:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:18Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d67723fff6d360 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: 4031df76-1e5c-49d6-8709-16cf5291ecb4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 3cdefa0f-50a9-40a3-8571-c90a98531b0e kind: Event lastTimestamp: "2026-09-18T16:28:21Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:28:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:21Z" name: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6.18d67724a02c9c7c namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: 4b20be28-a2a2-4fda-9244-40885bc386aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-191f9-predictor-6f8b8449b5 namespace: kserve-ci-e2e-test resourceVersion: "14893" uid: b321f8da-8322-4bc8-a4f9-1742292c738c kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: 'Created pod: success-200-isvc-191f9-predictor-6f8b8449b5-shdc6' metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:26:37Z" name: success-200-isvc-191f9-predictor-6f8b8449b5.18d6770c7678d482 namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 1820466e-7e89-493c-b295-bee70f070458 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-191f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14891" uid: e23a9b31-3fb0-4d66-9458-3cb3bcb4b888 kind: Event lastTimestamp: "2026-09-18T16:26:37Z" message: Scaled up replica set success-200-isvc-191f9-predictor-6f8b8449b5 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:26:37Z" name: success-200-isvc-191f9-predictor.18d6770c74d0f530 namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: 312de8e6-c8fb-4069-bcae-2104f56fd7ad reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-191f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: 239afe52-61a4-4e1c-b839-57c25bc2a7d1 kind: Event lastTimestamp: "2026-09-18T16:27:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:27:37Z" name: success-200-isvc-191f9-predictor.18d6770ff362a8cb namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 9f238950-0842-4708-8ee3-f7c9e99ac023 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-191f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: 239afe52-61a4-4e1c-b839-57c25bc2a7d1 kind: Event lastTimestamp: "2026-09-18T16:27:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:27:37Z" name: success-200-isvc-191f9-predictor.18d6770ff3c92049 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 699ae5fc-8d2c-4159-89b1-60a5487ba7ba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:27:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-191f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: 239afe52-61a4-4e1c-b839-57c25bc2a7d1 kind: Event lastTimestamp: "2026-09-18T16:28:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:07Z" name: success-200-isvc-191f9-predictor.18d6771deed84c26 namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8eb711e7-84e5-40d1-a748-716d2432b0b3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:27:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-191f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: 239afe52-61a4-4e1c-b839-57c25bc2a7d1 kind: Event lastTimestamp: "2026-09-18T16:28:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:07Z" name: success-200-isvc-191f9-predictor.18d6771def21e25a namespace: kserve-ci-e2e-test resourceVersion: "15871" uid: af1b0a99-f434-4702-b635-c6e40130b62a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:26:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-191f9 namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: 4bddc184-43c9-42ee-9c21-6e7f6c319c75 kind: Event lastTimestamp: "2026-09-18T16:27:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:26:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:27:56Z" name: success-200-isvc-191f9.18d6770c74d5297f namespace: kserve-ci-e2e-test resourceVersion: "15763" uid: c51b7407-e5e3-4625-a26e-36d16e899964 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:27:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-191f9 namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 4bddc184-43c9-42ee-9c21-6e7f6c319c75 kind: Event lastTimestamp: "2026-09-18T16:27:56Z" message: InferenceService [success-200-isvc-191f9] is Ready metadata: creationTimestamp: "2026-09-18T16:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:27:56Z" name: success-200-isvc-191f9.18d6771ed3874896 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: f3254ae6-f431-4de6-92a6-8cf2483f29d9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:37:43.672016Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6304f-predictor-9c8948987-8lbnb namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 613372f2-24b2-4f97-950d-198bc835eeb1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6304f-predictor-9c8948987-8lbnb to ip-10-0-135-110.ec2.internal metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:37:43Z" name: success-200-isvc-6304f-predictor-9c8948987-8lbnb.18d677a79b1df48c namespace: kserve-ci-e2e-test resourceVersion: "20520" uid: adceceab-b8b1-4ddd-9d08-d5f8e14589ec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:37:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6304f-predictor-9c8948987-8lbnb namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 613372f2-24b2-4f97-950d-198bc835eeb1 kind: Event lastTimestamp: "2026-09-18T16:37:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6304f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:44Z" name: success-200-isvc-6304f-predictor-9c8948987-8lbnb.18d677a7aa12a745 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 215139d8-8cd7-43ad-8a1a-1a0c9743d390 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6304f-predictor-9c8948987-8lbnb namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 613372f2-24b2-4f97-950d-198bc835eeb1 kind: Event lastTimestamp: "2026-09-18T16:37:45Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:37:45Z" name: success-200-isvc-6304f-predictor-9c8948987-8lbnb.18d677a81659c0c8 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: c4aa6af1-57d6-4d5f-9639-ee6185b8bef9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6304f-predictor-9c8948987-8lbnb namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 613372f2-24b2-4f97-950d-198bc835eeb1 kind: Event lastTimestamp: "2026-09-18T16:37:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:45Z" name: success-200-isvc-6304f-predictor-9c8948987-8lbnb.18d677a817ce20e8 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: d4a14bda-6c75-4b63-a57d-3c5daa399bd3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6304f-predictor-9c8948987-8lbnb namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 613372f2-24b2-4f97-950d-198bc835eeb1 kind: Event lastTimestamp: "2026-09-18T16:37:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:45Z" name: success-200-isvc-6304f-predictor-9c8948987-8lbnb.18d677a81b72b212 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 48718314-c447-438e-a5a2-4a53dbc77a4d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6304f-predictor-9c8948987-8lbnb namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 613372f2-24b2-4f97-950d-198bc835eeb1 kind: Event lastTimestamp: "2026-09-18T16:37:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:45Z" name: success-200-isvc-6304f-predictor-9c8948987-8lbnb.18d677a81bf6aec1 namespace: kserve-ci-e2e-test resourceVersion: "20596" uid: c169876c-2f14-4db5-a17e-8fc4e04bd237 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6304f-predictor-9c8948987-8lbnb namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 613372f2-24b2-4f97-950d-198bc835eeb1 kind: Event lastTimestamp: "2026-09-18T16:37:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:45Z" name: success-200-isvc-6304f-predictor-9c8948987-8lbnb.18d677a81c028df4 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 475b9fa1-d88f-4765-bfc9-bcd6a4172490 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6304f-predictor-9c8948987-8lbnb namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 613372f2-24b2-4f97-950d-198bc835eeb1 kind: Event lastTimestamp: "2026-09-18T16:37:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:45Z" name: success-200-isvc-6304f-predictor-9c8948987-8lbnb.18d677a81ee83839 namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: 802ffb2d-66be-463a-9daf-cb7d0120293e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6304f-predictor-9c8948987-8lbnb namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 613372f2-24b2-4f97-950d-198bc835eeb1 kind: Event lastTimestamp: "2026-09-18T16:37:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:45Z" name: success-200-isvc-6304f-predictor-9c8948987-8lbnb.18d677a81f5f83da namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: 5b99ab6d-98b8-40a9-8ff0-b46339d1bfc5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:37:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6304f-predictor-9c8948987-8lbnb namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 613372f2-24b2-4f97-950d-198bc835eeb1 kind: Event lastTimestamp: "2026-09-18T16:38:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:37:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:32Z" name: success-200-isvc-6304f-predictor-9c8948987-8lbnb.18d677a85b926a5e namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: 042f30aa-d7b7-49a7-86ca-bdbdb79489bc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6304f-predictor-9c8948987-8lbnb namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 613372f2-24b2-4f97-950d-198bc835eeb1 kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:04Z" name: success-200-isvc-6304f-predictor-9c8948987-8lbnb.18d677ba63d4f1eb namespace: kserve-ci-e2e-test resourceVersion: "21464" uid: ec44a2c2-bce1-4284-b8df-7cc3e8472a2f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6304f-predictor-9c8948987-8lbnb namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 613372f2-24b2-4f97-950d-198bc835eeb1 kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:04Z" name: success-200-isvc-6304f-predictor-9c8948987-8lbnb.18d677ba63d551d0 namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 93729c5f-90da-4f41-8eb8-9eec9e26d096 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6304f-predictor-9c8948987-8lbnb namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 613372f2-24b2-4f97-950d-198bc835eeb1 kind: Event lastTimestamp: "2026-09-18T16:39:07Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:07Z" name: success-200-isvc-6304f-predictor-9c8948987-8lbnb.18d677bb37fcb772 namespace: kserve-ci-e2e-test resourceVersion: "21603" uid: d69e4272-86cb-4ccc-ba8c-d10b7deb9d32 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6304f-predictor-9c8948987 namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: 92a1d7bd-2ed0-4da9-be3b-dbd2aa83d9d6 kind: Event lastTimestamp: "2026-09-18T16:37:43Z" message: 'Created pod: success-200-isvc-6304f-predictor-9c8948987-8lbnb' metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:37:43Z" name: success-200-isvc-6304f-predictor-9c8948987.18d677a79abc01f4 namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: 8406b621-9a6c-4529-a8d9-d2bd0e8e4d38 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6304f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 22c116bc-b85f-4914-aa2b-3ac989cc6c2f kind: Event lastTimestamp: "2026-09-18T16:37:43Z" message: Scaled up replica set success-200-isvc-6304f-predictor-9c8948987 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:37:43Z" name: success-200-isvc-6304f-predictor.18d677a79791c7df namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: e31bec59-310f-4695-9e94-9b5cbd1334ba reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:37:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6304f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: 7160fce2-9f7c-45f5-97aa-0df84f22535d kind: Event lastTimestamp: "2026-09-18T16:38:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:37:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:13Z" name: success-200-isvc-6304f-predictor.18d677ab161c8f7f namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: c3cd5b66-9ebf-4776-9899-c3e7c9935bac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:37:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6304f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: 7160fce2-9f7c-45f5-97aa-0df84f22535d kind: Event lastTimestamp: "2026-09-18T16:38:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:37:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:13Z" name: success-200-isvc-6304f-predictor.18d677ab165c8cc4 namespace: kserve-ci-e2e-test resourceVersion: "20825" uid: 430c063b-05bb-4ab8-97d2-ea66195e5dc9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6304f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: 7160fce2-9f7c-45f5-97aa-0df84f22535d kind: Event lastTimestamp: "2026-09-18T16:38:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:58Z" name: success-200-isvc-6304f-predictor.18d677b213bb2d22 namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 56ccd672-7539-4fc7-be30-22b6971cae97 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:38:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6304f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: 7160fce2-9f7c-45f5-97aa-0df84f22535d kind: Event lastTimestamp: "2026-09-18T16:38:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:58Z" name: success-200-isvc-6304f-predictor.18d677b2141b04a7 namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: 233d7535-e054-4d06-b050-15adacaec230 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:37:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6304f namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 5ab53823-f524-4d4d-90f8-e980678c35c6 kind: Event lastTimestamp: "2026-09-18T16:38:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:43Z" name: success-200-isvc-6304f.18d677a79794ee75 namespace: kserve-ci-e2e-test resourceVersion: "21224" uid: 342ea72c-c7b1-4939-b3db-a33e3315cf8a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6304f namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: 5ab53823-f524-4d4d-90f8-e980678c35c6 kind: Event lastTimestamp: "2026-09-18T16:38:42Z" message: InferenceService [success-200-isvc-6304f] is Ready metadata: creationTimestamp: "2026-09-18T16:38:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:42Z" name: success-200-isvc-6304f.18d677b56b0db614 namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 85c26d13-5c04-4fbf-86a7-49e597e63eed reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:39:45.911287Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-630d7-predictor-76777f589-z5cbj namespace: kserve-ci-e2e-test resourceVersion: "22030" uid: 8f93bdc0-729b-42b5-9d61-d7eff0a48438 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-630d7-predictor-76777f589-z5cbj to ip-10-0-135-110.ec2.internal metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:39:45Z" name: success-200-isvc-630d7-predictor-76777f589-z5cbj.18d677c4112535a4 namespace: kserve-ci-e2e-test resourceVersion: "22035" uid: a1023c4f-5cd1-4afb-a743-aae190b9739c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-630d7-predictor-76777f589-z5cbj namespace: kserve-ci-e2e-test resourceVersion: "22041" uid: 8f93bdc0-729b-42b5-9d61-d7eff0a48438 kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:39:46Z" name: success-200-isvc-630d7-predictor-76777f589-z5cbj.18d677c43368e5ab namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: 3b3521d1-145d-4418-b248-1023e3950427 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-630d7-predictor-76777f589-z5cbj namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 8f93bdc0-729b-42b5-9d61-d7eff0a48438 kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:46Z" name: success-200-isvc-630d7-predictor-76777f589-z5cbj.18d677c434ac57f6 namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: 6241fb9a-0aca-4b4d-957f-d3178406a716 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-630d7-predictor-76777f589-z5cbj namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 8f93bdc0-729b-42b5-9d61-d7eff0a48438 kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:46Z" name: success-200-isvc-630d7-predictor-76777f589-z5cbj.18d677c43780c9db namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: b9f92f58-b18b-48bf-b2f2-4b18b28935de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-630d7-predictor-76777f589-z5cbj namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 8f93bdc0-729b-42b5-9d61-d7eff0a48438 kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:46Z" name: success-200-isvc-630d7-predictor-76777f589-z5cbj.18d677c437ed09f3 namespace: kserve-ci-e2e-test resourceVersion: "22082" uid: 471a819b-cfeb-4c39-9e8b-1e3718edac66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-630d7-predictor-76777f589-z5cbj namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 8f93bdc0-729b-42b5-9d61-d7eff0a48438 kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:46Z" name: success-200-isvc-630d7-predictor-76777f589-z5cbj.18d677c437f7d354 namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: c729e36d-6c19-4986-b538-86e23078b8cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-630d7-predictor-76777f589-z5cbj namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 8f93bdc0-729b-42b5-9d61-d7eff0a48438 kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:46Z" name: success-200-isvc-630d7-predictor-76777f589-z5cbj.18d677c43b6a85a3 namespace: kserve-ci-e2e-test resourceVersion: "22084" uid: e6a7a088-ea50-46fc-8dda-f663728462e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-630d7-predictor-76777f589-z5cbj namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 8f93bdc0-729b-42b5-9d61-d7eff0a48438 kind: Event lastTimestamp: "2026-09-18T16:39:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:46Z" name: success-200-isvc-630d7-predictor-76777f589-z5cbj.18d677c43bd9a6f0 namespace: kserve-ci-e2e-test resourceVersion: "22085" uid: 44ebd067-3174-4364-8426-95ed24f39090 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T16:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-630d7-predictor-76777f589-z5cbj namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 8f93bdc0-729b-42b5-9d61-d7eff0a48438 kind: Event lastTimestamp: "2026-09-18T16:49:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:13Z" name: success-200-isvc-630d7-predictor-76777f589-z5cbj.18d677c46b626a65 namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: 655613a5-3495-4e7e-ac9f-02db52571e79 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-630d7-predictor-76777f589-z5cbj namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 8f93bdc0-729b-42b5-9d61-d7eff0a48438 kind: Event lastTimestamp: "2026-09-18T16:49:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:10Z" name: success-200-isvc-630d7-predictor-76777f589-z5cbj.18d678479e97b923 namespace: kserve-ci-e2e-test resourceVersion: "26657" uid: bc8712b8-166a-48ea-b73f-82c162c03698 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-630d7-predictor-76777f589-z5cbj namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 8f93bdc0-729b-42b5-9d61-d7eff0a48438 kind: Event lastTimestamp: "2026-09-18T16:49:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:10Z" name: success-200-isvc-630d7-predictor-76777f589-z5cbj.18d678479e9825dd namespace: kserve-ci-e2e-test resourceVersion: "26662" uid: 66c491f6-7882-4b5a-96b5-1bdd6db1d3d6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-630d7-predictor-76777f589-z5cbj namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: 8f93bdc0-729b-42b5-9d61-d7eff0a48438 kind: Event lastTimestamp: "2026-09-18T16:49:13Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:49:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:13Z" name: success-200-isvc-630d7-predictor-76777f589-z5cbj.18d6784833cf1231 namespace: kserve-ci-e2e-test resourceVersion: "26773" uid: 71bb7e41-fdf9-4d4e-b14c-510cc884b7ab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-630d7-predictor-76777f589 namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: c2f59fb0-723b-40c8-afb9-575839038704 kind: Event lastTimestamp: "2026-09-18T16:39:45Z" message: 'Created pod: success-200-isvc-630d7-predictor-76777f589-z5cbj' metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:45Z" name: success-200-isvc-630d7-predictor-76777f589.18d677c410ba84a2 namespace: kserve-ci-e2e-test resourceVersion: "22031" uid: ee53072e-6d62-4b66-87c4-71ecf756c0d3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-630d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: b9e42156-d0b6-456b-89ce-d6be3c870be5 kind: Event lastTimestamp: "2026-09-18T16:39:45Z" message: Scaled up replica set success-200-isvc-630d7-predictor-76777f589 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:45Z" name: success-200-isvc-630d7-predictor.18d677c40f05da51 namespace: kserve-ci-e2e-test resourceVersion: "22025" uid: 7d9d71c3-268d-4aa3-9f77-a1032156ba2a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:40:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-630d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22023" uid: e5869d4d-f427-4c71-9335-6e6a7f9996d3 kind: Event lastTimestamp: "2026-09-18T16:40:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:15Z" name: success-200-isvc-630d7-predictor.18d677c78d9b1d6f namespace: kserve-ci-e2e-test resourceVersion: "22392" uid: 25a4cee8-9600-4999-8df9-ac5d0440ec1e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:40:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-630d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22023" uid: e5869d4d-f427-4c71-9335-6e6a7f9996d3 kind: Event lastTimestamp: "2026-09-18T16:40:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:40:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:15Z" name: success-200-isvc-630d7-predictor.18d677c78e0e9f75 namespace: kserve-ci-e2e-test resourceVersion: "22393" uid: d624736e-070f-4883-a3ed-ca30d90f9a53 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:40:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-630d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22235" uid: e5869d4d-f427-4c71-9335-6e6a7f9996d3 kind: Event lastTimestamp: "2026-09-18T16:41:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:41:00Z" name: success-200-isvc-630d7-predictor.18d677ce8c542e3c namespace: kserve-ci-e2e-test resourceVersion: "22794" uid: b97c06c9-6e53-4c02-acfa-32f82b19f48f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:40:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-630d7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22235" uid: e5869d4d-f427-4c71-9335-6e6a7f9996d3 kind: Event lastTimestamp: "2026-09-18T16:41:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:41:00Z" name: success-200-isvc-630d7-predictor.18d677ce8cc5afd9 namespace: kserve-ci-e2e-test resourceVersion: "22795" uid: d0c5f3c1-1c4b-49ed-a59d-882d870c337e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-630d7 namespace: kserve-ci-e2e-test resourceVersion: "22011" uid: ba40a3f1-f7a8-4147-a487-66593fa1e2d0 kind: Event lastTimestamp: "2026-09-18T16:40:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:40:43Z" name: success-200-isvc-630d7.18d677c40f0950f5 namespace: kserve-ci-e2e-test resourceVersion: "22624" uid: bcd51b00-1a6a-44a7-8a13-dd1acc3b1fcf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-630d7 namespace: kserve-ci-e2e-test resourceVersion: "22622" uid: ba40a3f1-f7a8-4147-a487-66593fa1e2d0 kind: Event lastTimestamp: "2026-09-18T16:40:43Z" message: InferenceService [success-200-isvc-630d7] is Ready metadata: creationTimestamp: "2026-09-18T16:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:40:43Z" name: success-200-isvc-630d7.18d677d17ac5f875 namespace: kserve-ci-e2e-test resourceVersion: "22623" uid: 8677bb72-f0eb-4f36-a222-a60af4f2b5ad reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:38:24.746158Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: 630df95e-f477-4d96-85a8-e21e8ceff156 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f to ip-10-0-135-110.ec2.internal metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:38:24Z" name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f.18d677b12b5399f3 namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: 59c22299-5630-4649-acf3-af0e81a8cf3c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 630df95e-f477-4d96-85a8-e21e8ceff156 kind: Event lastTimestamp: "2026-09-18T16:38:25Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:38:25Z" name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f.18d677b14d545647 namespace: kserve-ci-e2e-test resourceVersion: "21027" uid: 6cc34903-2245-4ba2-b112-1159adb18dbf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 630df95e-f477-4d96-85a8-e21e8ceff156 kind: Event lastTimestamp: "2026-09-18T16:38:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:25Z" name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f.18d677b14ebb0496 namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: b30a3a9d-885d-4208-8c20-0ed9a44dd16b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 630df95e-f477-4d96-85a8-e21e8ceff156 kind: Event lastTimestamp: "2026-09-18T16:38:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:25Z" name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f.18d677b15199ad39 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 05a109b0-5b13-4068-a472-de53445398f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 630df95e-f477-4d96-85a8-e21e8ceff156 kind: Event lastTimestamp: "2026-09-18T16:38:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:25Z" name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f.18d677b15207ea14 namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 77fd9cb1-771b-4a6c-867a-938f27209ddb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 630df95e-f477-4d96-85a8-e21e8ceff156 kind: Event lastTimestamp: "2026-09-18T16:38:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:25Z" name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f.18d677b15211629c namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 50f94e58-514c-4ec1-b9d2-ddda307ac850 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 630df95e-f477-4d96-85a8-e21e8ceff156 kind: Event lastTimestamp: "2026-09-18T16:38:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:25Z" name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f.18d677b154db1c51 namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: d44a36da-5963-4e15-a74b-ee439946c003 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 630df95e-f477-4d96-85a8-e21e8ceff156 kind: Event lastTimestamp: "2026-09-18T16:38:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:25Z" name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f.18d677b155474d77 namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: 5daccc10-8f7f-4cf2-84e0-fef05c7f6540 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:38:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 630df95e-f477-4d96-85a8-e21e8ceff156 kind: Event lastTimestamp: "2026-09-18T16:39:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:38:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:13Z" name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f.18d677b1b67885b5 namespace: kserve-ci-e2e-test resourceVersion: "21650" uid: bfa0ec05-60c0-4ca1-8291-a168d2c9f3f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 630df95e-f477-4d96-85a8-e21e8ceff156 kind: Event lastTimestamp: "2026-09-18T16:39:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:45Z" name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f.18d677c409aba8ae namespace: kserve-ci-e2e-test resourceVersion: "22000" uid: b9ad4599-e8d2-4869-bfe4-1063edd58875 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 630df95e-f477-4d96-85a8-e21e8ceff156 kind: Event lastTimestamp: "2026-09-18T16:39:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:45Z" name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f.18d677c409ac2d38 namespace: kserve-ci-e2e-test resourceVersion: "22006" uid: 98bdbfbe-4820-438e-8476-8f163b990561 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 630df95e-f477-4d96-85a8-e21e8ceff156 kind: Event lastTimestamp: "2026-09-18T16:39:48Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:48Z" name: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f.18d677c492d11374 namespace: kserve-ci-e2e-test resourceVersion: "22111" uid: 70d1792d-62e7-47dd-9ba5-b7dca14bee97 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7c9ed-predictor-7fbffd8ccc namespace: kserve-ci-e2e-test resourceVersion: "20966" uid: f4473887-0a90-4508-9fdd-98d1f1fe57fe kind: Event lastTimestamp: "2026-09-18T16:38:24Z" message: 'Created pod: success-200-isvc-7c9ed-predictor-7fbffd8ccc-xzt2f' metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:24Z" name: success-200-isvc-7c9ed-predictor-7fbffd8ccc.18d677b12abe17df namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 6412a404-d579-43c6-bebd-d832da1df656 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7c9ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: 2ab52b58-3ac4-4ee8-9225-f910c94ec7fd kind: Event lastTimestamp: "2026-09-18T16:38:24Z" message: Scaled up replica set success-200-isvc-7c9ed-predictor-7fbffd8ccc from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:24Z" name: success-200-isvc-7c9ed-predictor.18d677b1285b6c6a namespace: kserve-ci-e2e-test resourceVersion: "20968" uid: 21669f51-785d-4938-b5ff-d483f8a76e8b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c9ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: 4824beca-4e1e-4ab2-8dea-3f1ab871cdda kind: Event lastTimestamp: "2026-09-18T16:38:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:39Z" name: success-200-isvc-7c9ed-predictor.18d677b4a6e6b07c namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: ff14a1fc-5c23-4584-b113-f98494d1778d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c9ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: 4824beca-4e1e-4ab2-8dea-3f1ab871cdda kind: Event lastTimestamp: "2026-09-18T16:38:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:38:39Z" name: success-200-isvc-7c9ed-predictor.18d677b4a7444778 namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: 4c8a6213-d57a-482a-9bb9-eedd7ff53aae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:38:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c9ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 4824beca-4e1e-4ab2-8dea-3f1ab871cdda kind: Event lastTimestamp: "2026-09-18T16:39:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:39Z" name: success-200-isvc-7c9ed-predictor.18d677b82672040e namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 42c21043-efea-467d-b675-ceb53691f71f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-18T16:38:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c9ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 4824beca-4e1e-4ab2-8dea-3f1ab871cdda kind: Event lastTimestamp: "2026-09-18T16:39:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:39Z" name: success-200-isvc-7c9ed-predictor.18d677b826d97412 namespace: kserve-ci-e2e-test resourceVersion: "21908" uid: 04561e48-fba1-4837-b59b-e72c92710bb2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c9ed namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: f8054bcd-6935-4a0f-a542-1ab2b9b15156 kind: Event lastTimestamp: "2026-09-18T16:39:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:23Z" name: success-200-isvc-7c9ed.18d677b128718279 namespace: kserve-ci-e2e-test resourceVersion: "21744" uid: e3fa5fba-7f01-4eb7-b413-915277cc6c07 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c9ed namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: f8054bcd-6935-4a0f-a542-1ab2b9b15156 kind: Event lastTimestamp: "2026-09-18T16:38:24Z" message: 'Failed to update status for InferenceService "success-200-isvc-7c9ed": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7c9ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:24Z" name: success-200-isvc-7c9ed.18d677b128fce3f6 namespace: kserve-ci-e2e-test resourceVersion: "20973" uid: 6ef375f8-1766-47a9-94d3-0ee715aa9d9d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c9ed namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: f8054bcd-6935-4a0f-a542-1ab2b9b15156 kind: Event lastTimestamp: "2026-09-18T16:38:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7c9ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:38:24Z" name: success-200-isvc-7c9ed.18d677b128fd2635 namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: d7924775-f760-41e9-a89f-49434e5fc18a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c9ed namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: f8054bcd-6935-4a0f-a542-1ab2b9b15156 kind: Event lastTimestamp: "2026-09-18T16:39:23Z" message: InferenceService [success-200-isvc-7c9ed] is Ready metadata: creationTimestamp: "2026-09-18T16:39:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:23Z" name: success-200-isvc-7c9ed.18d677bec647441b namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 0e3507c2-0bf5-4c64-a841-214923cac09a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:28:18.582215Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 3354167c-5adc-4697-a501-a3985a5aa4c6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8aa04-predictor-b58b89b85-5tmrg to ip-10-0-135-110.ec2.internal metadata: creationTimestamp: "2026-09-18T16:28:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:28:18Z" name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg.18d677240923e6a3 namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: cee7afef-afd4-41d8-b06a-35db4b8d361c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 3354167c-5adc-4697-a501-a3985a5aa4c6 kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:28:19Z" name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg.18d677242b4f3c5e namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: c36ca468-a53c-42f6-85ac-02b5f7842846 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 3354167c-5adc-4697-a501-a3985a5aa4c6 kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg.18d677242cb6be78 namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 57a098c2-aa0a-4720-b815-fd5c4e1c0f52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 3354167c-5adc-4697-a501-a3985a5aa4c6 kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg.18d677242f7d0344 namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: 5eeaf66e-60a4-4ce2-87c0-c2a5b61a68e0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 3354167c-5adc-4697-a501-a3985a5aa4c6 kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg.18d677242fe39dc6 namespace: kserve-ci-e2e-test resourceVersion: "16099" uid: 1d2e0480-9413-4f18-924f-cb5c847b40e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 3354167c-5adc-4697-a501-a3985a5aa4c6 kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg.18d677242fedb2fb namespace: kserve-ci-e2e-test resourceVersion: "16101" uid: 82b5de76-4aaa-486e-b1b7-c1cbc71a4121 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 3354167c-5adc-4697-a501-a3985a5aa4c6 kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg.18d677243305176c namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: 1d5e7ab9-f466-4461-a5ce-fe37e58b17b1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 3354167c-5adc-4697-a501-a3985a5aa4c6 kind: Event lastTimestamp: "2026-09-18T16:28:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:19Z" name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg.18d67724335fdfc4 namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: 474086dc-8b5d-447f-a329-91389b1ae27b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T16:28:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 3354167c-5adc-4697-a501-a3985a5aa4c6 kind: Event lastTimestamp: "2026-09-18T16:37:45Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:45Z" name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg.18d67724403b4f56 namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: 403a4a48-14ed-4cf7-aa13-473c95aa7747 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 3354167c-5adc-4697-a501-a3985a5aa4c6 kind: Event lastTimestamp: "2026-09-18T16:37:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:43Z" name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg.18d677a792b08883 namespace: kserve-ci-e2e-test resourceVersion: "20488" uid: b915b44f-83dc-475f-afa2-3ab8daa52183 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 3354167c-5adc-4697-a501-a3985a5aa4c6 kind: Event lastTimestamp: "2026-09-18T16:37:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:43Z" name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg.18d677a792b10633 namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: ae2eafbb-096b-4b97-bf57-469972ab2901 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 3354167c-5adc-4697-a501-a3985a5aa4c6 kind: Event lastTimestamp: "2026-09-18T16:37:45Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:45Z" name: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg.18d677a808b5d69c namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: 0b7562ef-ae1c-439a-804e-c6d106240e98 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8aa04-predictor-b58b89b85 namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: a3b5637f-9880-43d4-a1ac-528303c96749 kind: Event lastTimestamp: "2026-09-18T16:28:18Z" message: 'Created pod: success-200-isvc-8aa04-predictor-b58b89b85-5tmrg' metadata: creationTimestamp: "2026-09-18T16:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:18Z" name: success-200-isvc-8aa04-predictor-b58b89b85.18d6772408af82ba namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: bd601dd9-e61d-4332-8e02-b5973e87043c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8aa04-predictor namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: dddbcef9-16e1-4534-8668-375576bb688a kind: Event lastTimestamp: "2026-09-18T16:28:18Z" message: Scaled up replica set success-200-isvc-8aa04-predictor-b58b89b85 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:18Z" name: success-200-isvc-8aa04-predictor.18d67724051fd978 namespace: kserve-ci-e2e-test resourceVersion: "16041" uid: c5551a6c-3f83-4c59-9a51-2a36d60a819a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:28:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8aa04-predictor namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: 15f82968-b82f-47bf-8669-1ef2e0c03263 kind: Event lastTimestamp: "2026-09-18T16:28:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:48Z" name: success-200-isvc-8aa04-predictor.18d6772783b86499 namespace: kserve-ci-e2e-test resourceVersion: "16424" uid: 35f14b79-c2a2-4105-b5d5-e0ee258eced3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:28:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8aa04-predictor namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: 15f82968-b82f-47bf-8669-1ef2e0c03263 kind: Event lastTimestamp: "2026-09-18T16:28:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:48Z" name: success-200-isvc-8aa04-predictor.18d6772783f173d8 namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: f3dbe46f-a2cd-4c0e-a1f4-ed099aeb9cc8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:29:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8aa04-predictor namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: 15f82968-b82f-47bf-8669-1ef2e0c03263 kind: Event lastTimestamp: "2026-09-18T16:29:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:29:33Z" name: success-200-isvc-8aa04-predictor.18d6772e820066b9 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: 750b194b-a12b-4ead-a925-1c860161b162 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:29:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8aa04-predictor namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: 15f82968-b82f-47bf-8669-1ef2e0c03263 kind: Event lastTimestamp: "2026-09-18T16:29:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:29:33Z" name: success-200-isvc-8aa04-predictor.18d6772e8259aef2 namespace: kserve-ci-e2e-test resourceVersion: "17016" uid: 4d2cc9b1-bfda-492b-9480-ee4ac1f68539 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:28:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8aa04 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 8d209f16-9e32-4a86-811e-b09919cd7104 kind: Event lastTimestamp: "2026-09-18T16:29:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:29:15Z" name: success-200-isvc-8aa04.18d677240525269e namespace: kserve-ci-e2e-test resourceVersion: "16842" uid: 1a9bdf19-cbf8-4623-8dc4-88cc1229c2dd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8aa04 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 8d209f16-9e32-4a86-811e-b09919cd7104 kind: Event lastTimestamp: "2026-09-18T16:29:15Z" message: InferenceService [success-200-isvc-8aa04] is Ready metadata: creationTimestamp: "2026-09-18T16:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:29:15Z" name: success-200-isvc-8aa04.18d6773150a09889 namespace: kserve-ci-e2e-test resourceVersion: "16840" uid: 3af9924d-819a-40f4-a273-4227726239fd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:28:59.071285Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: 0bd6b51b-e04b-4862-9519-14b4e16b9a70 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8d780-predictor-577499d7ff-2nw5r to ip-10-0-135-110.ec2.internal metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:28:59Z" name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r.18d6772d767a0eb9 namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: bb0a14cb-82af-4498-8dbf-9e5f41cc71b3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: 0bd6b51b-e04b-4862-9519-14b4e16b9a70 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:28:59Z" name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r.18d6772d98616ba4 namespace: kserve-ci-e2e-test resourceVersion: "16657" uid: 802af606-ef7f-48f6-ad2a-11d69538ea4a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 0bd6b51b-e04b-4862-9519-14b4e16b9a70 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:59Z" name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r.18d6772d99b5fb51 namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: 934aa1fb-d8a8-48d1-8aab-396d0ee0b2d2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 0bd6b51b-e04b-4862-9519-14b4e16b9a70 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:59Z" name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r.18d6772d9c79135f namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: 52157b64-3ace-41be-b568-1429f7f384f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 0bd6b51b-e04b-4862-9519-14b4e16b9a70 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:59Z" name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r.18d6772d9d7e1126 namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: 3892cee7-12a7-4a6c-ac65-7dd53effe98e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 0bd6b51b-e04b-4862-9519-14b4e16b9a70 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:59Z" name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r.18d6772d9d87ae97 namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: 56499c6c-c0fc-4dc2-a102-37604d3ac876 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 0bd6b51b-e04b-4862-9519-14b4e16b9a70 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:59Z" name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r.18d6772da0065191 namespace: kserve-ci-e2e-test resourceVersion: "16663" uid: f6696ec8-72f1-4317-9a91-940d9fe0c069 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 0bd6b51b-e04b-4862-9519-14b4e16b9a70 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:59Z" name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r.18d6772da0886ffd namespace: kserve-ci-e2e-test resourceVersion: "16664" uid: 4dfbd7e4-75dc-4626-91df-80431b1e817e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T16:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 0bd6b51b-e04b-4862-9519-14b4e16b9a70 kind: Event lastTimestamp: "2026-09-18T16:38:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:27Z" name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r.18d6772e1568230a namespace: kserve-ci-e2e-test resourceVersion: "21073" uid: 0bddd158-df88-4ea8-baaa-72d07a611ddb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 0bd6b51b-e04b-4862-9519-14b4e16b9a70 kind: Event lastTimestamp: "2026-09-18T16:38:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:24Z" name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r.18d677b12523295e namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: 80474d6f-9cc5-4667-818f-4b9a509d5cb2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 0bd6b51b-e04b-4862-9519-14b4e16b9a70 kind: Event lastTimestamp: "2026-09-18T16:38:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:24Z" name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r.18d677b125242c8f namespace: kserve-ci-e2e-test resourceVersion: "20943" uid: bbfe3bf1-5927-4174-98e4-5c5fc8acceb1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:38:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 0bd6b51b-e04b-4862-9519-14b4e16b9a70 kind: Event lastTimestamp: "2026-09-18T16:38:27Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:38:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:27Z" name: success-200-isvc-8d780-predictor-577499d7ff-2nw5r.18d677b1dde5b907 namespace: kserve-ci-e2e-test resourceVersion: "21072" uid: 3df6fa5f-e68e-43c2-ac84-6433e5df30d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8d780-predictor-577499d7ff namespace: kserve-ci-e2e-test resourceVersion: "16601" uid: 54791d1d-b176-43a8-91b0-c3eebd0173b9 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: 'Created pod: success-200-isvc-8d780-predictor-577499d7ff-2nw5r' metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:59Z" name: success-200-isvc-8d780-predictor-577499d7ff.18d6772d75cf4ef6 namespace: kserve-ci-e2e-test resourceVersion: "16609" uid: 7c4ecf2f-1aa9-44dc-b600-9b95ee0a1a65 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8d780-predictor namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: 5b5ecf6c-b384-43d7-92fd-42f0c7867890 kind: Event lastTimestamp: "2026-09-18T16:28:59Z" message: Scaled up replica set success-200-isvc-8d780-predictor-577499d7ff from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:59Z" name: success-200-isvc-8d780-predictor.18d6772d7409756d namespace: kserve-ci-e2e-test resourceVersion: "16602" uid: a61926cd-d97a-41ba-9a64-5be2cce5763b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:29:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d780-predictor namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: a6422ef9-f8c0-4a8c-ad5c-b2caab666984 kind: Event lastTimestamp: "2026-09-18T16:29:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:29:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:29:29Z" name: success-200-isvc-8d780-predictor.18d67730f2c61bbf namespace: kserve-ci-e2e-test resourceVersion: "16969" uid: f20456d1-ae46-472f-934f-929551a5ed52 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:29:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d780-predictor namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: a6422ef9-f8c0-4a8c-ad5c-b2caab666984 kind: Event lastTimestamp: "2026-09-18T16:29:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:29:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:29:29Z" name: success-200-isvc-8d780-predictor.18d67730f33b55d7 namespace: kserve-ci-e2e-test resourceVersion: "16970" uid: 2d47c727-efaf-44f2-a462-16065c4b61d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:29:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d780-predictor namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: a6422ef9-f8c0-4a8c-ad5c-b2caab666984 kind: Event lastTimestamp: "2026-09-18T16:30:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:30:14Z" name: success-200-isvc-8d780-predictor.18d67737f2a8417a namespace: kserve-ci-e2e-test resourceVersion: "17374" uid: 5589f091-71fa-4ec7-9051-2f13febc4eac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:29:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d780-predictor namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: a6422ef9-f8c0-4a8c-ad5c-b2caab666984 kind: Event lastTimestamp: "2026-09-18T16:30:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:30:14Z" name: success-200-isvc-8d780-predictor.18d67737f31472f1 namespace: kserve-ci-e2e-test resourceVersion: "17375" uid: c7e1cd25-b9e9-4638-bd2f-dc1568d189a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:28:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d780 namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: 899ac78a-0574-46b3-aa30-a1284cd261dc kind: Event lastTimestamp: "2026-09-18T16:29:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:29:57Z" name: success-200-isvc-8d780.18d6772d740d757f namespace: kserve-ci-e2e-test resourceVersion: "17218" uid: d007bcfe-cfc7-4e6b-aeae-0b8963a0aab9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d780 namespace: kserve-ci-e2e-test resourceVersion: "17215" uid: 899ac78a-0574-46b3-aa30-a1284cd261dc kind: Event lastTimestamp: "2026-09-18T16:29:57Z" message: InferenceService [success-200-isvc-8d780] is Ready metadata: creationTimestamp: "2026-09-18T16:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:29:57Z" name: success-200-isvc-8d780.18d6773b26974b25 namespace: kserve-ci-e2e-test resourceVersion: "17217" uid: b2f0df80-4b76-4b24-a783-9db25115573b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:49:11.066655Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26 namespace: kserve-ci-e2e-test resourceVersion: "26688" uid: 53c2ad28-054c-4eeb-9b70-1136a0c5cdfd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a2696-predictor-6d48d4c789-h4v26 to ip-10-0-135-110.ec2.internal metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:49:11Z" name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26.18d67847a707adf2 namespace: kserve-ci-e2e-test resourceVersion: "26694" uid: b22cebd2-c16d-40c5-bed6-f34c0b21b9c6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26 namespace: kserve-ci-e2e-test resourceVersion: "26698" uid: 53c2ad28-054c-4eeb-9b70-1136a0c5cdfd kind: Event lastTimestamp: "2026-09-18T16:49:11Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:49:11Z" name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26.18d67847c8f151f1 namespace: kserve-ci-e2e-test resourceVersion: "26736" uid: f1e48ca4-5ef6-4fce-afee-da50a1972035 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26 namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 53c2ad28-054c-4eeb-9b70-1136a0c5cdfd kind: Event lastTimestamp: "2026-09-18T16:49:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:11Z" name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26.18d67847ca4e5c78 namespace: kserve-ci-e2e-test resourceVersion: "26738" uid: facf06fd-b118-42fa-996d-519f3c3fc655 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26 namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 53c2ad28-054c-4eeb-9b70-1136a0c5cdfd kind: Event lastTimestamp: "2026-09-18T16:49:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:11Z" name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26.18d67847cd07abbf namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: 8bb8bc8a-bf2f-4377-bc53-9d3f77e19950 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26 namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 53c2ad28-054c-4eeb-9b70-1136a0c5cdfd kind: Event lastTimestamp: "2026-09-18T16:49:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:11Z" name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26.18d67847cd7cf53e namespace: kserve-ci-e2e-test resourceVersion: "26740" uid: 497d1f63-0e07-4f61-9cee-ed4faa35be28 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26 namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 53c2ad28-054c-4eeb-9b70-1136a0c5cdfd kind: Event lastTimestamp: "2026-09-18T16:49:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:11Z" name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26.18d67847cd8a01c9 namespace: kserve-ci-e2e-test resourceVersion: "26741" uid: 397636b2-5a1f-42ba-a420-d7b9e656ac5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26 namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 53c2ad28-054c-4eeb-9b70-1136a0c5cdfd kind: Event lastTimestamp: "2026-09-18T16:49:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:11Z" name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26.18d67847d03639b5 namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: 9fb26801-6c8c-4488-8bd2-d7b7e27ba1e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26 namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 53c2ad28-054c-4eeb-9b70-1136a0c5cdfd kind: Event lastTimestamp: "2026-09-18T16:49:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:11Z" name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26.18d67847d0a87a16 namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: dbdc68bc-beed-47ca-ab06-b04c82a2518a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26 namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 53c2ad28-054c-4eeb-9b70-1136a0c5cdfd kind: Event lastTimestamp: "2026-09-18T16:49:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:57Z" name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26.18d67847d29de5f9 namespace: kserve-ci-e2e-test resourceVersion: "27342" uid: 60e7c90f-20ed-48a6-973c-58a9373ee16b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26 namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 53c2ad28-054c-4eeb-9b70-1136a0c5cdfd kind: Event lastTimestamp: "2026-09-18T17:06:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T17:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:41Z" name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26.18d6793c2b00c088 namespace: kserve-ci-e2e-test resourceVersion: "34484" uid: 200a400b-ebd4-488d-bc39-39f3334731ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26 namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 53c2ad28-054c-4eeb-9b70-1136a0c5cdfd kind: Event lastTimestamp: "2026-09-18T17:06:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T17:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:41Z" name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26.18d6793c2b01ff81 namespace: kserve-ci-e2e-test resourceVersion: "34485" uid: 2e98e2eb-b525-4274-b391-f8c28b4ac5ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26 namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 53c2ad28-054c-4eeb-9b70-1136a0c5cdfd kind: Event lastTimestamp: "2026-09-18T17:06:42Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T17:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:42Z" name: success-200-isvc-a2696-predictor-6d48d4c789-h4v26.18d6793c878c6124 namespace: kserve-ci-e2e-test resourceVersion: "34545" uid: 8c3b8ac1-224e-42ea-a2af-bd399c34cf93 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a2696-predictor-6d48d4c789 namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 76ed1f83-8db6-4719-b544-ebb356c69ec5 kind: Event lastTimestamp: "2026-09-18T16:49:11Z" message: 'Created pod: success-200-isvc-a2696-predictor-6d48d4c789-h4v26' metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:11Z" name: success-200-isvc-a2696-predictor-6d48d4c789.18d67847a6709967 namespace: kserve-ci-e2e-test resourceVersion: "26690" uid: 766e2587-b70f-44e8-8ad0-b3cd06cb6e0c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a2696-predictor namespace: kserve-ci-e2e-test resourceVersion: "26680" uid: 7dbbd290-fb94-42ef-a93c-17a984963d1d kind: Event lastTimestamp: "2026-09-18T16:49:11Z" message: Scaled up replica set success-200-isvc-a2696-predictor-6d48d4c789 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:11Z" name: success-200-isvc-a2696-predictor.18d67847a464864f namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: aced1923-688d-4bbe-be27-9440d90c3501 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:49:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a2696-predictor namespace: kserve-ci-e2e-test resourceVersion: "26681" uid: d314dce2-90f0-49dc-b70b-d4cc8be2975c kind: Event lastTimestamp: "2026-09-18T16:50:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:50:11Z" name: success-200-isvc-a2696-predictor.18d6784b2392db86 namespace: kserve-ci-e2e-test resourceVersion: "27470" uid: 188785c2-2b3a-4ec8-a794-e18d197ba599 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:49:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a2696-predictor namespace: kserve-ci-e2e-test resourceVersion: "26681" uid: d314dce2-90f0-49dc-b70b-d4cc8be2975c kind: Event lastTimestamp: "2026-09-18T16:50:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:50:11Z" name: success-200-isvc-a2696-predictor.18d6784b23faecc9 namespace: kserve-ci-e2e-test resourceVersion: "27471" uid: ed5110d9-a82c-4346-88b6-e4d094bb9274 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a2696-predictor namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: d314dce2-90f0-49dc-b70b-d4cc8be2975c kind: Event lastTimestamp: "2026-09-18T16:49:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:41Z" name: success-200-isvc-a2696-predictor.18d6784ea2b734b5 namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 9e6ada2b-91e9-4659-9a59-5c76633b73bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a2696-predictor namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: d314dce2-90f0-49dc-b70b-d4cc8be2975c kind: Event lastTimestamp: "2026-09-18T16:49:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:49:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:41Z" name: success-200-isvc-a2696-predictor.18d6784ea31c9e46 namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: ac89ed5d-0c60-44b0-9b23-e461998e855c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:49:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a2696 namespace: kserve-ci-e2e-test resourceVersion: "26667" uid: a3d0d4c8-1f75-4f73-b8d2-6b05f6424560 kind: Event lastTimestamp: "2026-09-18T16:50:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:50:07Z" name: success-200-isvc-a2696.18d67847a47dcb0a namespace: kserve-ci-e2e-test resourceVersion: "27435" uid: d59c4e49-a057-48a3-8992-94776bc5b38b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:50:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a2696 namespace: kserve-ci-e2e-test resourceVersion: "27433" uid: a3d0d4c8-1f75-4f73-b8d2-6b05f6424560 kind: Event lastTimestamp: "2026-09-18T16:50:07Z" message: InferenceService [success-200-isvc-a2696] is Ready metadata: creationTimestamp: "2026-09-18T16:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:50:07Z" name: success-200-isvc-a2696.18d67854e27180f1 namespace: kserve-ci-e2e-test resourceVersion: "27434" uid: 71c49f4f-fc5b-478c-9265-db84e5dcc32f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:48:29.711903Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: 1f9b2d18-1bfc-4ceb-a344-738bd027e59a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q to ip-10-0-135-110.ec2.internal metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:48:29Z" name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q.18d6783e061841df namespace: kserve-ci-e2e-test resourceVersion: "26231" uid: 839910a2-7dcf-4188-8bed-475cdcf0acf8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q namespace: kserve-ci-e2e-test resourceVersion: "26244" uid: 1f9b2d18-1bfc-4ceb-a344-738bd027e59a kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:48:30Z" name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q.18d6783e283b5ec3 namespace: kserve-ci-e2e-test resourceVersion: "26277" uid: ad6df12a-a1b2-47ae-ae22-2d687d3e6037 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: 1f9b2d18-1bfc-4ceb-a344-738bd027e59a kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:30Z" name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q.18d6783e29941e40 namespace: kserve-ci-e2e-test resourceVersion: "26280" uid: 9193cc41-5d64-4db2-884c-d433273fe01e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: 1f9b2d18-1bfc-4ceb-a344-738bd027e59a kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:30Z" name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q.18d6783e2caaab18 namespace: kserve-ci-e2e-test resourceVersion: "26281" uid: 03e922e6-60be-4d30-a752-054ac77b2677 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: 1f9b2d18-1bfc-4ceb-a344-738bd027e59a kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:30Z" name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q.18d6783e2d2328db namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: 63ef5331-06db-41ea-9e28-3f033de74c3f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: 1f9b2d18-1bfc-4ceb-a344-738bd027e59a kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:30Z" name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q.18d6783e2d2e2f8a namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: e9755bdb-9ed5-410c-9b88-e09c62c6bd0e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: 1f9b2d18-1bfc-4ceb-a344-738bd027e59a kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:30Z" name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q.18d6783e302fa8c7 namespace: kserve-ci-e2e-test resourceVersion: "26287" uid: 7069f452-eaf4-45c4-b353-a8d4763df238 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: 1f9b2d18-1bfc-4ceb-a344-738bd027e59a kind: Event lastTimestamp: "2026-09-18T16:48:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:30Z" name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q.18d6783e30aeb425 namespace: kserve-ci-e2e-test resourceVersion: "26288" uid: a55298b6-28d2-4637-805a-4fbc27cb8544 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-18T16:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: 1f9b2d18-1bfc-4ceb-a344-738bd027e59a kind: Event lastTimestamp: "2026-09-18T16:49:16Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:16Z" name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q.18d6783e3bbca4a1 namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: cd944db4-6c38-4e92-b498-182acefd1a55 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: 1f9b2d18-1bfc-4ceb-a344-738bd027e59a kind: Event lastTimestamp: "2026-09-18T16:49:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:50Z" name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q.18d67850ecde6bed namespace: kserve-ci-e2e-test resourceVersion: "27175" uid: 747bc99a-d6da-46e8-8caa-3e32c6aafc77 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: 1f9b2d18-1bfc-4ceb-a344-738bd027e59a kind: Event lastTimestamp: "2026-09-18T16:49:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:50Z" name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q.18d67850ecdf28ce namespace: kserve-ci-e2e-test resourceVersion: "27178" uid: 02035b06-e428-41bf-a7c6-e8f90b14ecec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: 1f9b2d18-1bfc-4ceb-a344-738bd027e59a kind: Event lastTimestamp: "2026-09-18T16:49:51Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:51Z" name: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q.18d67851182f399f namespace: kserve-ci-e2e-test resourceVersion: "27258" uid: 797ebfe9-9c34-43a2-a111-50cebef2be1d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b3de6-predictor-5f84b875fc namespace: kserve-ci-e2e-test resourceVersion: "26206" uid: b1981a2e-2ddf-4a6d-8ae7-5da96e11978d kind: Event lastTimestamp: "2026-09-18T16:48:29Z" message: 'Created pod: success-200-isvc-b3de6-predictor-5f84b875fc-wwp5q' metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:29Z" name: success-200-isvc-b3de6-predictor-5f84b875fc.18d6783e055b1ae5 namespace: kserve-ci-e2e-test resourceVersion: "26224" uid: cf5bcdfe-ddb7-4786-99aa-d846c8d3fca0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b3de6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26200" uid: 8f95277d-491f-4185-8cb1-0cec24242712 kind: Event lastTimestamp: "2026-09-18T16:48:29Z" message: Scaled up replica set success-200-isvc-b3de6-predictor-5f84b875fc from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:29Z" name: success-200-isvc-b3de6-predictor.18d6783e01cdb07a namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: a3dc0a8b-61b4-4a70-bbe6-30e3bcd0b3b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:48:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b3de6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26204" uid: 05b62c76-68a0-43f0-ba6f-a67d849ffa11 kind: Event lastTimestamp: "2026-09-18T16:48:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:59Z" name: success-200-isvc-b3de6-predictor.18d67841809607e9 namespace: kserve-ci-e2e-test resourceVersion: "26531" uid: eaa7d7bd-8190-46d0-b117-b17f7595da12 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:48:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b3de6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26204" uid: 05b62c76-68a0-43f0-ba6f-a67d849ffa11 kind: Event lastTimestamp: "2026-09-18T16:48:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:48:59Z" name: success-200-isvc-b3de6-predictor.18d678418130ae76 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: b72b9632-a127-463d-94b2-273ce27c6bdf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:49:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b3de6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26419" uid: 05b62c76-68a0-43f0-ba6f-a67d849ffa11 kind: Event lastTimestamp: "2026-09-18T16:49:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:44Z" name: success-200-isvc-b3de6-predictor.18d678487ec6a150 namespace: kserve-ci-e2e-test resourceVersion: "27088" uid: f1abe570-770f-4f69-bd64-d880f4ad8fc6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:49:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b3de6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26419" uid: 05b62c76-68a0-43f0-ba6f-a67d849ffa11 kind: Event lastTimestamp: "2026-09-18T16:49:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:44Z" name: success-200-isvc-b3de6-predictor.18d678487f3b5a44 namespace: kserve-ci-e2e-test resourceVersion: "27089" uid: 64c1a028-6010-4df2-bad3-2dbc346b3e1c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:48:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b3de6 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: dc16c175-58cb-4a12-b372-3b0ec720ea73 kind: Event lastTimestamp: "2026-09-18T16:49:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:49:26Z" name: success-200-isvc-b3de6.18d6783e021ac973 namespace: kserve-ci-e2e-test resourceVersion: "26900" uid: 0cb7e0e2-1bf7-4267-8522-8f4f35227318 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b3de6 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: dc16c175-58cb-4a12-b372-3b0ec720ea73 kind: Event lastTimestamp: "2026-09-18T16:48:29Z" message: 'Failed to update status for InferenceService "success-200-isvc-b3de6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b3de6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:48:29Z" name: success-200-isvc-b3de6.18d6783e028516f9 namespace: kserve-ci-e2e-test resourceVersion: "26212" uid: bd4e4960-34e5-4787-85e9-4970e187c627 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b3de6 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: dc16c175-58cb-4a12-b372-3b0ec720ea73 kind: Event lastTimestamp: "2026-09-18T16:48:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b3de6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:48:29Z" name: success-200-isvc-b3de6.18d6783e028565c4 namespace: kserve-ci-e2e-test resourceVersion: "26215" uid: 46a5e765-8bbd-46f3-82f0-666e9c5df332 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b3de6 namespace: kserve-ci-e2e-test resourceVersion: "26898" uid: dc16c175-58cb-4a12-b372-3b0ec720ea73 kind: Event lastTimestamp: "2026-09-18T16:49:26Z" message: InferenceService [success-200-isvc-b3de6] is Ready metadata: creationTimestamp: "2026-09-18T16:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:49:26Z" name: success-200-isvc-b3de6.18d6784b4b3faaf7 namespace: kserve-ci-e2e-test resourceVersion: "26899" uid: f2cbd812-054e-4357-a956-3bc06d1600ff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:49:51.024341Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf namespace: kserve-ci-e2e-test resourceVersion: "27207" uid: 91e82968-e9e7-472c-9412-0277920c713c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf to ip-10-0-135-110.ec2.internal metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:49:51Z" name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf.18d67850f4b1b214 namespace: kserve-ci-e2e-test resourceVersion: "27212" uid: f40e8c50-6101-423f-9fd3-1fd802385b63 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf namespace: kserve-ci-e2e-test resourceVersion: "27217" uid: 91e82968-e9e7-472c-9412-0277920c713c kind: Event lastTimestamp: "2026-09-18T16:49:51Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:49:51Z" name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf.18d6785116ab447e namespace: kserve-ci-e2e-test resourceVersion: "27255" uid: 52a5ff12-66e3-43c2-bf99-bc5b02af156c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: 91e82968-e9e7-472c-9412-0277920c713c kind: Event lastTimestamp: "2026-09-18T16:49:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:51Z" name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf.18d67851180c4eb2 namespace: kserve-ci-e2e-test resourceVersion: "27257" uid: 56dbb3e5-0838-4d2c-a0d4-c0064f771730 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: 91e82968-e9e7-472c-9412-0277920c713c kind: Event lastTimestamp: "2026-09-18T16:49:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:51Z" name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf.18d678511b7d1444 namespace: kserve-ci-e2e-test resourceVersion: "27259" uid: a1e68a63-9236-4708-a183-fbc1907c77e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: 91e82968-e9e7-472c-9412-0277920c713c kind: Event lastTimestamp: "2026-09-18T16:49:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:51Z" name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf.18d678511becbc1f namespace: kserve-ci-e2e-test resourceVersion: "27260" uid: b09cbd40-f4c7-4190-ba6c-3590f0538677 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: 91e82968-e9e7-472c-9412-0277920c713c kind: Event lastTimestamp: "2026-09-18T16:49:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:51Z" name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf.18d678511bf5bd4c namespace: kserve-ci-e2e-test resourceVersion: "27261" uid: 557c33f4-353c-4b34-98e7-47119b829670 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: 91e82968-e9e7-472c-9412-0277920c713c kind: Event lastTimestamp: "2026-09-18T16:49:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:51Z" name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf.18d678511eb45958 namespace: kserve-ci-e2e-test resourceVersion: "27262" uid: 8bec9404-c418-4774-adbf-88da1bc5292a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: 91e82968-e9e7-472c-9412-0277920c713c kind: Event lastTimestamp: "2026-09-18T16:49:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:49:51Z" name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf.18d678511f1fbaf1 namespace: kserve-ci-e2e-test resourceVersion: "27263" uid: 4c9d725a-ea56-4b0c-ac10-700de26d6179 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T16:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: 91e82968-e9e7-472c-9412-0277920c713c kind: Event lastTimestamp: "2026-09-18T16:59:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:59:18Z" name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf.18d6785169759d71 namespace: kserve-ci-e2e-test resourceVersion: "31454" uid: 38754887-f451-488f-a758-42ba84cb50fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:59:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: 91e82968-e9e7-472c-9412-0277920c713c kind: Event lastTimestamp: "2026-09-18T16:59:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:59:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:59:16Z" name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf.18d678d48b77c22f namespace: kserve-ci-e2e-test resourceVersion: "31421" uid: 34537494-503d-4ace-8e54-8c53ab23ea76 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:59:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: 91e82968-e9e7-472c-9412-0277920c713c kind: Event lastTimestamp: "2026-09-18T16:59:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:59:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:59:16Z" name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf.18d678d48b783b23 namespace: kserve-ci-e2e-test resourceVersion: "31424" uid: b2199949-0c46-441a-8fe9-2c3a9682e307 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:59:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: 91e82968-e9e7-472c-9412-0277920c713c kind: Event lastTimestamp: "2026-09-18T16:59:18Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:59:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:59:18Z" name: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf.18d678d531e64cd0 namespace: kserve-ci-e2e-test resourceVersion: "31453" uid: a83641c3-a850-4ade-8213-351f8a35370b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bb388-predictor-7fcb976d9f namespace: kserve-ci-e2e-test resourceVersion: "27201" uid: 5d25c836-1660-4f8c-885a-5b9ad343b75e kind: Event lastTimestamp: "2026-09-18T16:49:51Z" message: 'Created pod: success-200-isvc-bb388-predictor-7fcb976d9f-p9fmf' metadata: creationTimestamp: "2026-09-18T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:51Z" name: success-200-isvc-bb388-predictor-7fcb976d9f.18d67850f41883f3 namespace: kserve-ci-e2e-test resourceVersion: "27208" uid: 6f690a3f-a4b5-431e-9529-a8a0ba74ab0d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:49:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bb388-predictor namespace: kserve-ci-e2e-test resourceVersion: "27199" uid: ad3925a9-d586-4945-a670-df4f491881d4 kind: Event lastTimestamp: "2026-09-18T16:49:50Z" message: Scaled up replica set success-200-isvc-bb388-predictor-7fcb976d9f from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:49:50Z" name: success-200-isvc-bb388-predictor.18d67850f22f4d05 namespace: kserve-ci-e2e-test resourceVersion: "27202" uid: 5e6b8486-d867-46e8-a8b8-a8cdec230eab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bb388-predictor namespace: kserve-ci-e2e-test resourceVersion: "27200" uid: ddc3cf6a-318b-4db2-a655-6912af8c32d6 kind: Event lastTimestamp: "2026-09-18T16:50:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:50:36Z" name: success-200-isvc-bb388-predictor.18d6785470d1a452 namespace: kserve-ci-e2e-test resourceVersion: "27704" uid: da166876-9e3c-4b99-ab28-fd2300d98dd3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bb388-predictor namespace: kserve-ci-e2e-test resourceVersion: "27200" uid: ddc3cf6a-318b-4db2-a655-6912af8c32d6 kind: Event lastTimestamp: "2026-09-18T16:50:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:50:36Z" name: success-200-isvc-bb388-predictor.18d67854710e5063 namespace: kserve-ci-e2e-test resourceVersion: "27705" uid: 797f0a1b-97bb-440d-a666-9be042deb697 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:50:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bb388-predictor namespace: kserve-ci-e2e-test resourceVersion: "27408" uid: ddc3cf6a-318b-4db2-a655-6912af8c32d6 kind: Event lastTimestamp: "2026-09-18T16:51:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:51:21Z" name: success-200-isvc-bb388-predictor.18d6785eee10b7c2 namespace: kserve-ci-e2e-test resourceVersion: "28094" uid: ee96e122-d77c-49d9-b6f5-d0ac1c2588bd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-18T16:50:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bb388-predictor namespace: kserve-ci-e2e-test resourceVersion: "27408" uid: ddc3cf6a-318b-4db2-a655-6912af8c32d6 kind: Event lastTimestamp: "2026-09-18T16:51:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:51:21Z" name: success-200-isvc-bb388-predictor.18d6785eee53b7dd namespace: kserve-ci-e2e-test resourceVersion: "28095" uid: 9a60781f-b845-4a8d-a867-af3ead91a026 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:49:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bb388 namespace: kserve-ci-e2e-test resourceVersion: "27190" uid: d92cec5b-0a65-46b5-9e15-6a7f8821d36d kind: Event lastTimestamp: "2026-09-18T16:50:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:50:49Z" name: success-200-isvc-bb388.18d67850f2322391 namespace: kserve-ci-e2e-test resourceVersion: "27812" uid: 38ea0596-ce98-4b59-8f0f-39dabd0c52fd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:50:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bb388 namespace: kserve-ci-e2e-test resourceVersion: "27809" uid: d92cec5b-0a65-46b5-9e15-6a7f8821d36d kind: Event lastTimestamp: "2026-09-18T16:50:49Z" message: InferenceService [success-200-isvc-bb388] is Ready metadata: creationTimestamp: "2026-09-18T16:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:50:49Z" name: success-200-isvc-bb388.18d6785e7deabc1d namespace: kserve-ci-e2e-test resourceVersion: "27811" uid: e281dbf9-0abe-4137-8bd4-3f939550ef86 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:39:04.428614Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x namespace: kserve-ci-e2e-test resourceVersion: "21498" uid: 687dbe5b-1b65-448e-9b49-577aa1a260cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-de495-predictor-6b9d94d467-z9z4x to ip-10-0-135-110.ec2.internal metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:39:04Z" name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x.18d677ba6895d392 namespace: kserve-ci-e2e-test resourceVersion: "21503" uid: 8c1e86b1-b7b4-454a-835a-1e31038cbb64 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x namespace: kserve-ci-e2e-test resourceVersion: "21507" uid: 687dbe5b-1b65-448e-9b49-577aa1a260cf kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:39:05Z" name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x.18d677ba8aded514 namespace: kserve-ci-e2e-test resourceVersion: "21548" uid: 668a836f-2b91-40e6-b88a-77b5af4b3a78 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 687dbe5b-1b65-448e-9b49-577aa1a260cf kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-2013" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:05Z" name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x.18d677ba8c2269bf namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: 05bfa6ed-5e5d-423b-a683-437c88b4a925 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 687dbe5b-1b65-448e-9b49-577aa1a260cf kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:05Z" name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x.18d677ba8f06dc96 namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: 11ba6160-c128-41a5-843d-55f563c935ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 687dbe5b-1b65-448e-9b49-577aa1a260cf kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:05Z" name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x.18d677ba8f74644d namespace: kserve-ci-e2e-test resourceVersion: "21553" uid: 045526fa-1075-4e72-a50f-6a339aafce05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 687dbe5b-1b65-448e-9b49-577aa1a260cf kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:05Z" name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x.18d677ba8f7da980 namespace: kserve-ci-e2e-test resourceVersion: "21554" uid: 712d059e-5bcd-436e-b980-e19c094fda7a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 687dbe5b-1b65-448e-9b49-577aa1a260cf kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:05Z" name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x.18d677ba926257b6 namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: c3bc8770-9b98-473c-8f48-8e80320416bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 687dbe5b-1b65-448e-9b49-577aa1a260cf kind: Event lastTimestamp: "2026-09-18T16:39:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:39:05Z" name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x.18d677ba92cc6e56 namespace: kserve-ci-e2e-test resourceVersion: "21556" uid: 7d401613-146f-4c99-a8f0-6f8c3759d28c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-18T16:39:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 687dbe5b-1b65-448e-9b49-577aa1a260cf kind: Event lastTimestamp: "2026-09-18T16:48:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:32Z" name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x.18d677bad50d0927 namespace: kserve-ci-e2e-test resourceVersion: "26318" uid: e4dd2de8-ea64-43b9-8fd5-f4ea07d51c8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 687dbe5b-1b65-448e-9b49-577aa1a260cf kind: Event lastTimestamp: "2026-09-18T16:48:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:29Z" name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x.18d6783e01152425 namespace: kserve-ci-e2e-test resourceVersion: "26202" uid: 4ab3f7cf-10cc-41d5-b9ae-4bc541dcc800 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 687dbe5b-1b65-448e-9b49-577aa1a260cf kind: Event lastTimestamp: "2026-09-18T16:48:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-18T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:29Z" name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x.18d6783e0115c6e8 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: e49f2865-2277-42bf-a263-204ca4b1c12b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:48:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 687dbe5b-1b65-448e-9b49-577aa1a260cf kind: Event lastTimestamp: "2026-09-18T16:48:32Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-18T16:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:48:32Z" name: success-200-isvc-de495-predictor-6b9d94d467-z9z4x.18d6783e9d7d794e namespace: kserve-ci-e2e-test resourceVersion: "26315" uid: e6b95784-12f8-4884-a16c-da9a76fdf480 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-110.ec2.internal source: component: kubelet host: ip-10-0-135-110.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-de495-predictor-6b9d94d467 namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: 15e40b18-cdb2-45be-be18-63462b875ed7 kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: 'Created pod: success-200-isvc-de495-predictor-6b9d94d467-z9z4x' metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:04Z" name: success-200-isvc-de495-predictor-6b9d94d467.18d677ba681c12c6 namespace: kserve-ci-e2e-test resourceVersion: "21499" uid: 2786697a-e867-4fbc-8a6a-5e147ce6696c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-de495-predictor namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: 768d58b1-5c2f-4e58-bc70-01580e081c98 kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: Scaled up replica set success-200-isvc-de495-predictor-6b9d94d467 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:04Z" name: success-200-isvc-de495-predictor.18d677ba661ce94e namespace: kserve-ci-e2e-test resourceVersion: "21487" uid: f97c571f-9466-43ce-8014-50362d3a4163 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:39:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de495-predictor namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: 9ff7217d-62c8-4c95-bee9-c8534e8ce8c3 kind: Event lastTimestamp: "2026-09-18T16:39:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:34Z" name: success-200-isvc-de495-predictor.18d677bde5532049 namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: 39185b62-62fe-4a3a-8428-314cc2750e80 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:39:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de495-predictor namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: 9ff7217d-62c8-4c95-bee9-c8534e8ce8c3 kind: Event lastTimestamp: "2026-09-18T16:39:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:39:34Z" name: success-200-isvc-de495-predictor.18d677bde5b8d783 namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: a49f7f14-abe4-49b4-97e0-6c2b45c01e5b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:39:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de495-predictor namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: 9ff7217d-62c8-4c95-bee9-c8534e8ce8c3 kind: Event lastTimestamp: "2026-09-18T16:40:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:49Z" name: success-200-isvc-de495-predictor.18d677c4e33718a3 namespace: kserve-ci-e2e-test resourceVersion: "22670" uid: 690dc03e-8af2-4d6f-9d14-449af9bdc458 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:39:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-de495-predictor namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: 9ff7217d-62c8-4c95-bee9-c8534e8ce8c3 kind: Event lastTimestamp: "2026-09-18T16:40:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-18T16:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:40:49Z" name: success-200-isvc-de495-predictor.18d677c4e39a7187 namespace: kserve-ci-e2e-test resourceVersion: "22671" uid: 01df4cc0-975c-4c5e-afde-f769288b2860 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de495 namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: 6492ee9a-9065-45d9-88b7-5e3cc3092c35 kind: Event lastTimestamp: "2026-09-18T16:40:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:40:02Z" name: success-200-isvc-de495.18d677ba664cb7b4 namespace: kserve-ci-e2e-test resourceVersion: "22257" uid: f46f4afb-6c4b-4fdc-a305-aeb205ffe497 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de495 namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: 6492ee9a-9065-45d9-88b7-5e3cc3092c35 kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-de495": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-de495": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:04Z" name: success-200-isvc-de495.18d677ba66b91db0 namespace: kserve-ci-e2e-test resourceVersion: "21492" uid: b58d70d5-a148-4837-8d2a-10827f9c38ab reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:39:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de495 namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: 6492ee9a-9065-45d9-88b7-5e3cc3092c35 kind: Event lastTimestamp: "2026-09-18T16:39:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-de495": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:39:04Z" name: success-200-isvc-de495.18d677ba66b95f32 namespace: kserve-ci-e2e-test resourceVersion: "21497" uid: f9e398ca-02c5-4995-afdc-c8531a7bf716 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:40:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-de495 namespace: kserve-ci-e2e-test resourceVersion: "22255" uid: 6492ee9a-9065-45d9-88b7-5e3cc3092c35 kind: Event lastTimestamp: "2026-09-18T16:40:02Z" message: InferenceService [success-200-isvc-de495] is Ready metadata: creationTimestamp: "2026-09-18T16:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:40:02Z" name: success-200-isvc-de495.18d677c7e4fcccbe namespace: kserve-ci-e2e-test resourceVersion: "22256" uid: f08cad3e-a398-4ce8-b4e9-88e860f2fa14 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:28:07.165043Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-191f9-58b7f9f45-q6mtt namespace: kserve-ci-e2e-test resourceVersion: "15849" uid: 68cb5412-9cb2-4cf8-9ab6-98467eea14f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-191f9-58b7f9f45-q6mtt to ip-10-0-128-224.ec2.internal metadata: creationTimestamp: "2026-09-18T16:28:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:28:07Z" name: switch-graph-191f9-58b7f9f45-q6mtt.18d67721609fb062 namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 71283873-3508-4926-8d47-cfd83ebec646 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:07Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-191f9-58b7f9f45-q6mtt namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 68cb5412-9cb2-4cf8-9ab6-98467eea14f1 kind: Event lastTimestamp: "2026-09-18T16:28:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-191f9-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:07Z" name: switch-graph-191f9-58b7f9f45-q6mtt.18d677216f5eea33 namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 41c3e689-803a-43d3-ad30-820b94ce35d6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:08Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-191f9-58b7f9f45-q6mtt namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: 68cb5412-9cb2-4cf8-9ab6-98467eea14f1 kind: Event lastTimestamp: "2026-09-18T16:28:08Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:28:08Z" name: switch-graph-191f9-58b7f9f45-q6mtt.18d67721a65117a2 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: 76a7ddb4-8b11-42b1-991a-cb2c3fc718d6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-191f9} kind: Pod name: switch-graph-191f9-58b7f9f45-q6mtt namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 68cb5412-9cb2-4cf8-9ab6-98467eea14f1 kind: Event lastTimestamp: "2026-09-18T16:28:08Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" metadata: creationTimestamp: "2026-09-18T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:08Z" name: switch-graph-191f9-58b7f9f45-q6mtt.18d67721a79b800a namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: f5b9ded6-afb0-4464-acf7-b4b0843aae26 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-191f9} kind: Pod name: switch-graph-191f9-58b7f9f45-q6mtt namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 68cb5412-9cb2-4cf8-9ab6-98467eea14f1 kind: Event lastTimestamp: "2026-09-18T16:28:10Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" in 2.366s (2.366s including waiting). Image size: 211751521 bytes.' metadata: creationTimestamp: "2026-09-18T16:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:10Z" name: switch-graph-191f9-58b7f9f45-q6mtt.18d6772234a80a33 namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: ae5d1b5b-18fb-4c0b-9253-011b7eb3b60a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-191f9} kind: Pod name: switch-graph-191f9-58b7f9f45-q6mtt namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 68cb5412-9cb2-4cf8-9ab6-98467eea14f1 kind: Event lastTimestamp: "2026-09-18T16:28:10Z" message: 'Created container: switch-graph-191f9' metadata: creationTimestamp: "2026-09-18T16:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:10Z" name: switch-graph-191f9-58b7f9f45-q6mtt.18d67722376e6c88 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: dae958ca-40aa-41f5-a0a4-504871df0991 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-191f9} kind: Pod name: switch-graph-191f9-58b7f9f45-q6mtt namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 68cb5412-9cb2-4cf8-9ab6-98467eea14f1 kind: Event lastTimestamp: "2026-09-18T16:28:10Z" message: Started container switch-graph-191f9 metadata: creationTimestamp: "2026-09-18T16:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:10Z" name: switch-graph-191f9-58b7f9f45-q6mtt.18d6772237e59d71 namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 94f2a0bf-3d72-40d0-a111-0ce061e8924b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-191f9} kind: Pod name: switch-graph-191f9-58b7f9f45-q6mtt namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 68cb5412-9cb2-4cf8-9ab6-98467eea14f1 kind: Event lastTimestamp: "2026-09-18T16:28:18Z" message: Stopping container switch-graph-191f9 metadata: creationTimestamp: "2026-09-18T16:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:18Z" name: switch-graph-191f9-58b7f9f45-q6mtt.18d67723f7e991ef namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: cf1861c4-f1f6-4908-b83f-b2a31f9bf9ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:28:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-191f9} kind: Pod name: switch-graph-191f9-58b7f9f45-q6mtt namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 68cb5412-9cb2-4cf8-9ab6-98467eea14f1 kind: Event lastTimestamp: "2026-09-18T16:28:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:28:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:28:46Z" name: switch-graph-191f9-58b7f9f45-q6mtt.18d67724d0a14f06 namespace: kserve-ci-e2e-test resourceVersion: "16376" uid: 4adfdd22-9f5d-4d0f-8da9-0a128a202d72 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-191f9-58b7f9f45 namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: ad78a41b-e1c8-46aa-95b7-9108c2c4bd85 kind: Event lastTimestamp: "2026-09-18T16:28:07Z" message: 'Created pod: switch-graph-191f9-58b7f9f45-q6mtt' metadata: creationTimestamp: "2026-09-18T16:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:07Z" name: switch-graph-191f9-58b7f9f45.18d67721602acdf2 namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: 2d423009-c715-4c95-a3e4-f5145b56f8d9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-191f9 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: ec9b547e-bde9-4b67-909d-edc7b3fa35cc kind: Event lastTimestamp: "2026-09-18T16:28:07Z" message: Scaled up replica set switch-graph-191f9-58b7f9f45 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:28:07Z" name: switch-graph-191f9.18d677215ef42b20 namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: e376cc2f-c28c-43a3-8e65-19c7a16d36ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-191f9 namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 7d371187-e962-4ec3-8a0e-9bca50729d39 kind: Event lastTimestamp: "2026-09-18T16:28:07Z" message: InferenceGraph [switch-graph-191f9] is Ready metadata: creationTimestamp: "2026-09-18T16:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:28:07Z" name: switch-graph-191f9.18d6772164a13353 namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: 584b2474-bda1-4e38-99f9-eb3ac89a334b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-191f9 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 7d371187-e962-4ec3-8a0e-9bca50729d39 kind: Event lastTimestamp: "2026-09-18T16:28:07Z" message: 'Failed to update status for InferenceGraph "switch-graph-191f9": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-191f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:28:07Z" name: switch-graph-191f9.18d6772166f48d89 namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: c9b4e6f0-7adb-4137-a55b-4b2a8e5b2686 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:28:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-191f9 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 7d371187-e962-4ec3-8a0e-9bca50729d39 kind: Event lastTimestamp: "2026-09-18T16:28:07Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-191f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-18T16:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:28:07Z" name: switch-graph-191f9.18d6772166f4d2c1 namespace: kserve-ci-e2e-test resourceVersion: "15864" uid: 58290743-c695-429f-a42c-515b7b28e1d3 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:29:28.574623Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8aa04-785cbdc5c8-rrcvw namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 567b00dd-701a-4510-8f0d-08117c3dd4a4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-8aa04-785cbdc5c8-rrcvw to ip-10-0-128-224.ec2.internal metadata: creationTimestamp: "2026-09-18T16:29:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:29:28Z" name: switch-graph-8aa04-785cbdc5c8-rrcvw.18d677345503400f namespace: kserve-ci-e2e-test resourceVersion: "16951" uid: 8c8aca6d-794f-4290-8bf3-13e01424e4b6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:28Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8aa04-785cbdc5c8-rrcvw namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 567b00dd-701a-4510-8f0d-08117c3dd4a4 kind: Event lastTimestamp: "2026-09-18T16:29:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-8aa04-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:29:28Z" name: switch-graph-8aa04-785cbdc5c8-rrcvw.18d6773464944fcc namespace: kserve-ci-e2e-test resourceVersion: "16962" uid: 1fc11574-16f3-40c4-bf0e-6f6a7c787932 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:29Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8aa04-785cbdc5c8-rrcvw namespace: kserve-ci-e2e-test resourceVersion: "16955" uid: 567b00dd-701a-4510-8f0d-08117c3dd4a4 kind: Event lastTimestamp: "2026-09-18T16:29:29Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:29:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:29:29Z" name: switch-graph-8aa04-785cbdc5c8-rrcvw.18d677349ac78b07 namespace: kserve-ci-e2e-test resourceVersion: "16980" uid: 0fded65e-192d-4264-9ac2-59b8ee852221 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8aa04} kind: Pod name: switch-graph-8aa04-785cbdc5c8-rrcvw namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 567b00dd-701a-4510-8f0d-08117c3dd4a4 kind: Event lastTimestamp: "2026-09-18T16:29:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" already present on machine metadata: creationTimestamp: "2026-09-18T16:29:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:29:29Z" name: switch-graph-8aa04-785cbdc5c8-rrcvw.18d677349cca61fd namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: 51a77b63-d5e2-4e26-b5b6-b5621c5cdc5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8aa04} kind: Pod name: switch-graph-8aa04-785cbdc5c8-rrcvw namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 567b00dd-701a-4510-8f0d-08117c3dd4a4 kind: Event lastTimestamp: "2026-09-18T16:29:29Z" message: 'Created container: switch-graph-8aa04' metadata: creationTimestamp: "2026-09-18T16:29:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:29:29Z" name: switch-graph-8aa04-785cbdc5c8-rrcvw.18d677349fba7e9d namespace: kserve-ci-e2e-test resourceVersion: "16983" uid: 93bd1aa2-f407-4593-8c24-ecb7bea849df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8aa04} kind: Pod name: switch-graph-8aa04-785cbdc5c8-rrcvw namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 567b00dd-701a-4510-8f0d-08117c3dd4a4 kind: Event lastTimestamp: "2026-09-18T16:29:29Z" message: Started container switch-graph-8aa04 metadata: creationTimestamp: "2026-09-18T16:29:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:29:29Z" name: switch-graph-8aa04-785cbdc5c8-rrcvw.18d67734a02b8b64 namespace: kserve-ci-e2e-test resourceVersion: "16984" uid: ca7d967e-6218-4214-bc23-18b17e70eade reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:37:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8aa04} kind: Pod name: switch-graph-8aa04-785cbdc5c8-rrcvw namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 567b00dd-701a-4510-8f0d-08117c3dd4a4 kind: Event lastTimestamp: "2026-09-18T16:37:43Z" message: Stopping container switch-graph-8aa04 metadata: creationTimestamp: "2026-09-18T16:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:37:43Z" name: switch-graph-8aa04-785cbdc5c8-rrcvw.18d677a78b0b5e0e namespace: kserve-ci-e2e-test resourceVersion: "20457" uid: 810d9078-6132-4de4-838a-b9bbeb16a612 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T16:37:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8aa04} kind: Pod name: switch-graph-8aa04-785cbdc5c8-rrcvw namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 567b00dd-701a-4510-8f0d-08117c3dd4a4 kind: Event lastTimestamp: "2026-09-18T16:38:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T16:37:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:38:11Z" name: switch-graph-8aa04-785cbdc5c8-rrcvw.18d677a83502179c namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 7018cf40-0f6f-4ce9-b1ea-218abc5a1e28 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-8aa04-785cbdc5c8 namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: 87805d8c-84ed-405a-82dd-b6a48b0806b7 kind: Event lastTimestamp: "2026-09-18T16:29:28Z" message: 'Created pod: switch-graph-8aa04-785cbdc5c8-rrcvw' metadata: creationTimestamp: "2026-09-18T16:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:29:28Z" name: switch-graph-8aa04-785cbdc5c8.18d677345498645c namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: badb8af9-bc54-433a-a274-c10dd3936132 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-8aa04 namespace: kserve-ci-e2e-test resourceVersion: "16941" uid: 6f43f711-7d51-4028-b20b-22972cb7419b kind: Event lastTimestamp: "2026-09-18T16:29:28Z" message: Scaled up replica set switch-graph-8aa04-785cbdc5c8 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:29:28Z" name: switch-graph-8aa04.18d67734536ddc17 namespace: kserve-ci-e2e-test resourceVersion: "16944" uid: 0c315ce9-020f-4815-a366-006d8c326bc3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:29:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-8aa04 namespace: kserve-ci-e2e-test resourceVersion: "16957" uid: 0d56b059-9920-497e-9397-9a2237acf282 kind: Event lastTimestamp: "2026-09-18T16:29:28Z" message: InferenceGraph [switch-graph-8aa04] is Ready metadata: creationTimestamp: "2026-09-18T16:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:29:28Z" name: switch-graph-8aa04.18d67734587d5925 namespace: kserve-ci-e2e-test resourceVersion: "16959" uid: db6975d5-ccf5-4ae3-abf3-d07fdf060449 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:29:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8aa04 namespace: kserve-ci-e2e-test resourceVersion: "16942" uid: bdf164f5-a2ad-427c-8c26-792f14a2598b kind: Event lastTimestamp: "2026-09-18T16:29:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:29:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:29:58Z" name: switch-graph-8aa04.18d67737d1c95a29 namespace: kserve-ci-e2e-test resourceVersion: "17222" uid: e5a194c2-f951-49e8-820e-efc26ad91b27 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:29:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8aa04 namespace: kserve-ci-e2e-test resourceVersion: "16942" uid: bdf164f5-a2ad-427c-8c26-792f14a2598b kind: Event lastTimestamp: "2026-09-18T16:29:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:29:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:29:58Z" name: switch-graph-8aa04.18d67737d22a2579 namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 6eb4423d-16b6-46f9-aa1e-a82212053ff7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-18T16:50:21.042721Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a2696-7758d57596-5hcnh namespace: kserve-ci-e2e-test resourceVersion: "27556" uid: cab18aeb-ddda-490a-95cc-509bb9e21939 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-a2696-7758d57596-5hcnh to ip-10-0-128-224.ec2.internal metadata: creationTimestamp: "2026-09-18T16:50:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-18T16:50:21Z" name: switch-graph-a2696-7758d57596-5hcnh.18d67857f1edcc39 namespace: kserve-ci-e2e-test resourceVersion: "27561" uid: 235425a0-7be0-45db-8a7f-2fbe6b6b21bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-694bcbb5ff-k6nv5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:50:21Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a2696-7758d57596-5hcnh namespace: kserve-ci-e2e-test resourceVersion: "27559" uid: cab18aeb-ddda-490a-95cc-509bb9e21939 kind: Event lastTimestamp: "2026-09-18T16:50:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-a2696-serving-cert" not found' metadata: creationTimestamp: "2026-09-18T16:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:50:21Z" name: switch-graph-a2696-7758d57596-5hcnh.18d67857f9fc91b3 namespace: kserve-ci-e2e-test resourceVersion: "27576" uid: f33892f5-7910-4dfb-a99d-df9be9ac15eb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:50:22Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a2696-7758d57596-5hcnh namespace: kserve-ci-e2e-test resourceVersion: "27565" uid: cab18aeb-ddda-490a-95cc-509bb9e21939 kind: Event lastTimestamp: "2026-09-18T16:50:22Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-18T16:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-18T16:50:22Z" name: switch-graph-a2696-7758d57596-5hcnh.18d6785837b4e743 namespace: kserve-ci-e2e-test resourceVersion: "27589" uid: c106f433-9457-48ef-b1d6-a6bc910cae5c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a2696} kind: Pod name: switch-graph-a2696-7758d57596-5hcnh namespace: kserve-ci-e2e-test resourceVersion: "27559" uid: cab18aeb-ddda-490a-95cc-509bb9e21939 kind: Event lastTimestamp: "2026-09-18T16:50:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9ee58bd289ecb7aa4010c311b867cae9a40a42edc53d096a1f0354150db5c48e" already present on machine metadata: creationTimestamp: "2026-09-18T16:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:50:22Z" name: switch-graph-a2696-7758d57596-5hcnh.18d6785838fcca77 namespace: kserve-ci-e2e-test resourceVersion: "27591" uid: d9840812-1b8b-4fc0-8402-1efc63b2096c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a2696} kind: Pod name: switch-graph-a2696-7758d57596-5hcnh namespace: kserve-ci-e2e-test resourceVersion: "27559" uid: cab18aeb-ddda-490a-95cc-509bb9e21939 kind: Event lastTimestamp: "2026-09-18T16:50:22Z" message: 'Created container: switch-graph-a2696' metadata: creationTimestamp: "2026-09-18T16:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:50:22Z" name: switch-graph-a2696-7758d57596-5hcnh.18d678583bac059c namespace: kserve-ci-e2e-test resourceVersion: "27593" uid: 34f281f1-f045-4094-997e-471808911d4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a2696} kind: Pod name: switch-graph-a2696-7758d57596-5hcnh namespace: kserve-ci-e2e-test resourceVersion: "27559" uid: cab18aeb-ddda-490a-95cc-509bb9e21939 kind: Event lastTimestamp: "2026-09-18T16:50:22Z" message: Started container switch-graph-a2696 metadata: creationTimestamp: "2026-09-18T16:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T16:50:22Z" name: switch-graph-a2696-7758d57596-5hcnh.18d678583c1b1824 namespace: kserve-ci-e2e-test resourceVersion: "27594" uid: 907937ce-cc2b-4d28-996f-442f968f9cb7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T17:06:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a2696} kind: Pod name: switch-graph-a2696-7758d57596-5hcnh namespace: kserve-ci-e2e-test resourceVersion: "27559" uid: cab18aeb-ddda-490a-95cc-509bb9e21939 kind: Event lastTimestamp: "2026-09-18T17:06:41Z" message: Stopping container switch-graph-a2696 metadata: creationTimestamp: "2026-09-18T17:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:06:41Z" name: switch-graph-a2696-7758d57596-5hcnh.18d6793c22d64427 namespace: kserve-ci-e2e-test resourceVersion: "34469" uid: b785f645-87eb-4287-848e-b88c25b45e6d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-18T17:06:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a2696} kind: Pod name: switch-graph-a2696-7758d57596-5hcnh namespace: kserve-ci-e2e-test resourceVersion: "27559" uid: cab18aeb-ddda-490a-95cc-509bb9e21939 kind: Event lastTimestamp: "2026-09-18T17:07:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-18T17:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-18T17:07:08Z" name: switch-graph-a2696-7758d57596-5hcnh.18d6793caa9561c5 namespace: kserve-ci-e2e-test resourceVersion: "34742" uid: 220c54b3-941a-4af5-8beb-34fccf1d01aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-224.ec2.internal source: component: kubelet host: ip-10-0-128-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:50:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-a2696-7758d57596 namespace: kserve-ci-e2e-test resourceVersion: "27551" uid: c444c6ef-5c41-49c4-8bbe-7bd192b7a5c0 kind: Event lastTimestamp: "2026-09-18T16:50:21Z" message: 'Created pod: switch-graph-a2696-7758d57596-5hcnh' metadata: creationTimestamp: "2026-09-18T16:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:50:21Z" name: switch-graph-a2696-7758d57596.18d67857f17de40b namespace: kserve-ci-e2e-test resourceVersion: "27558" uid: d5113f24-6715-4490-bfd1-da05f30cf313 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:50:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-a2696 namespace: kserve-ci-e2e-test resourceVersion: "27549" uid: 398c4e00-2f5e-42e2-8b4f-b3082d714569 kind: Event lastTimestamp: "2026-09-18T16:50:21Z" message: Scaled up replica set switch-graph-a2696-7758d57596 from 0 to 1 metadata: creationTimestamp: "2026-09-18T16:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:50:21Z" name: switch-graph-a2696.18d67857f0811b11 namespace: kserve-ci-e2e-test resourceVersion: "27553" uid: ef31f305-68b1-4d33-bf27-b8e874f480cf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-18T16:50:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-a2696 namespace: kserve-ci-e2e-test resourceVersion: "27568" uid: 694d1ada-705f-4e61-b916-4fbc5f315c06 kind: Event lastTimestamp: "2026-09-18T16:50:21Z" message: InferenceGraph [switch-graph-a2696] is Ready metadata: creationTimestamp: "2026-09-18T16:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-18T16:50:21Z" name: switch-graph-a2696.18d67857f41d3e22 namespace: kserve-ci-e2e-test resourceVersion: "27570" uid: 9ea250cc-0ea0-4516-abc9-4e7aed584541 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:50:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a2696 namespace: kserve-ci-e2e-test resourceVersion: "27550" uid: 0c5d9cb6-e19d-4ab1-b107-0a7a768a9e30 kind: Event lastTimestamp: "2026-09-18T16:50:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:50:51Z" name: switch-graph-a2696.18d6785b6ef0bd34 namespace: kserve-ci-e2e-test resourceVersion: "27839" uid: 6935b943-baa0-4ccb-8e07-5b32961ea83e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-18T16:50:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a2696 namespace: kserve-ci-e2e-test resourceVersion: "27550" uid: 0c5d9cb6-e19d-4ab1-b107-0a7a768a9e30 kind: Event lastTimestamp: "2026-09-18T16:50:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-18T16:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-18T16:50:51Z" name: switch-graph-a2696.18d6785b6f5b3a02 namespace: kserve-ci-e2e-test resourceVersion: "27841" uid: c313a827-d819-47b1-bc92-37b887349e26 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34747"