--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:12:00.737788Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: 3305d51f-389c-485b-bbde-94986e6daa53 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-78ee8-85bfbcfdb7-hzl6c to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:12:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:12:00Z" name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c.18b92c60898ab4a2 namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: 196bd5f3-165d-4ccf-b05c-0f3e2b7dbff9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:01Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c namespace: kserve-ci-e2e-test resourceVersion: "21553" uid: 3305d51f-389c-485b-bbde-94986e6daa53 kind: Event lastTimestamp: "2026-06-15T06:12:01Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:12:01Z" name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c.18b92c60a396897a namespace: kserve-ci-e2e-test resourceVersion: "21570" uid: ac94cea2-8b01-47aa-9311-4bdc53a55d65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-78ee8} kind: Pod name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c namespace: kserve-ci-e2e-test resourceVersion: "21546" uid: 3305d51f-389c-485b-bbde-94986e6daa53 kind: Event lastTimestamp: "2026-06-15T06:12:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" already present on machine metadata: creationTimestamp: "2026-06-15T06:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:01Z" name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c.18b92c60a4f4ec79 namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: b133baa4-8ce8-43d9-8497-c6e41748c5a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-78ee8} kind: Pod name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c namespace: kserve-ci-e2e-test resourceVersion: "21546" uid: 3305d51f-389c-485b-bbde-94986e6daa53 kind: Event lastTimestamp: "2026-06-15T06:12:01Z" message: 'Created container: ensemble-graph-78ee8' metadata: creationTimestamp: "2026-06-15T06:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:01Z" name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c.18b92c60a870f50e namespace: kserve-ci-e2e-test resourceVersion: "21573" uid: 47580017-410d-4b5e-a1b8-8433ea2e244c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-78ee8} kind: Pod name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c namespace: kserve-ci-e2e-test resourceVersion: "21546" uid: 3305d51f-389c-485b-bbde-94986e6daa53 kind: Event lastTimestamp: "2026-06-15T06:12:01Z" message: Started container ensemble-graph-78ee8 metadata: creationTimestamp: "2026-06-15T06:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:01Z" name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c.18b92c60a918a201 namespace: kserve-ci-e2e-test resourceVersion: "21575" uid: a2af6240-7cb0-4f47-a96c-1a2e8998a84a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-78ee8} kind: Pod name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c namespace: kserve-ci-e2e-test resourceVersion: "21546" uid: 3305d51f-389c-485b-bbde-94986e6daa53 kind: Event lastTimestamp: "2026-06-15T06:12:10Z" message: Stopping container ensemble-graph-78ee8 metadata: creationTimestamp: "2026-06-15T06:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:10Z" name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c.18b92c62e1aa7156 namespace: kserve-ci-e2e-test resourceVersion: "21669" uid: 1695b969-1e8d-4782-b681-d3c0f709b5b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-78ee8} kind: Pod name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c namespace: kserve-ci-e2e-test resourceVersion: "21546" uid: 3305d51f-389c-485b-bbde-94986e6daa53 kind: Event lastTimestamp: "2026-06-15T06:12:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:37Z" name: ensemble-graph-78ee8-85bfbcfdb7-hzl6c.18b92c6344dd48fd namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: 2a5c8eb5-4c0a-40a0-83cf-b93ff36620a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-78ee8-85bfbcfdb7 namespace: kserve-ci-e2e-test resourceVersion: "21541" uid: c45bce19-d958-4525-b6f1-ceb9a062ef0c kind: Event lastTimestamp: "2026-06-15T06:12:00Z" message: 'Created pod: ensemble-graph-78ee8-85bfbcfdb7-hzl6c' metadata: creationTimestamp: "2026-06-15T06:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:00Z" name: ensemble-graph-78ee8-85bfbcfdb7.18b92c6089111600 namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: a559c971-2647-497b-b313-ec156c74dcf5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-78ee8 namespace: kserve-ci-e2e-test resourceVersion: "21538" uid: 233acc49-956e-45d8-b347-99d3b7f7c514 kind: Event lastTimestamp: "2026-06-15T06:12:00Z" message: Scaled up replica set ensemble-graph-78ee8-85bfbcfdb7 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:00Z" name: ensemble-graph-78ee8.18b92c6088524922 namespace: kserve-ci-e2e-test resourceVersion: "21542" uid: c56466bf-55c3-4c83-8504-95a50f9bd837 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-78ee8 namespace: kserve-ci-e2e-test resourceVersion: "21556" uid: 45c56256-4198-4d67-bfaf-23b1aafc6c33 kind: Event lastTimestamp: "2026-06-15T06:12:00Z" message: InferenceGraph [ensemble-graph-78ee8] is Ready metadata: creationTimestamp: "2026-06-15T06:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:00Z" name: ensemble-graph-78ee8.18b92c608d9d36e6 namespace: kserve-ci-e2e-test resourceVersion: "21558" uid: 6463c175-4eb6-4fda-8223-a653edefe5d8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-78ee8 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: 45c56256-4198-4d67-bfaf-23b1aafc6c33 kind: Event lastTimestamp: "2026-06-15T06:12:00Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-78ee8": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-78ee8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:00Z" name: ensemble-graph-78ee8.18b92c6090a27dc3 namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: 5efb250c-7d96-4449-be15-236a2a6a9873 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-78ee8 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: 45c56256-4198-4d67-bfaf-23b1aafc6c33 kind: Event lastTimestamp: "2026-06-15T06:12:00Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-78ee8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:00Z" name: ensemble-graph-78ee8.18b92c6090a2c8b0 namespace: kserve-ci-e2e-test resourceVersion: "21563" uid: 4262c1bd-26da-4569-a82b-5a200e4c3392 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:13:11.001212Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b50b1-97587847f-nkhzb namespace: kserve-ci-e2e-test resourceVersion: "22515" uid: 2c2e122f-76c2-49e6-8401-8af1f72d500f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-b50b1-97587847f-nkhzb to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:13:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:13:11Z" name: ensemble-graph-b50b1-97587847f-nkhzb.18b92c70e5918362 namespace: kserve-ci-e2e-test resourceVersion: "22520" uid: 85c47d0f-69bf-4a43-aaee-86597f706912 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:11Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b50b1-97587847f-nkhzb namespace: kserve-ci-e2e-test resourceVersion: "22524" uid: 2c2e122f-76c2-49e6-8401-8af1f72d500f kind: Event lastTimestamp: "2026-06-15T06:13:11Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:13:11Z" name: ensemble-graph-b50b1-97587847f-nkhzb.18b92c70ffddfec3 namespace: kserve-ci-e2e-test resourceVersion: "22537" uid: a74721ad-42e8-482c-b766-7c2ce233807b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b50b1} kind: Pod name: ensemble-graph-b50b1-97587847f-nkhzb namespace: kserve-ci-e2e-test resourceVersion: "22517" uid: 2c2e122f-76c2-49e6-8401-8af1f72d500f kind: Event lastTimestamp: "2026-06-15T06:13:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" already present on machine metadata: creationTimestamp: "2026-06-15T06:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:11Z" name: ensemble-graph-b50b1-97587847f-nkhzb.18b92c71011cda4f namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: b5f7a691-9859-4dfb-8660-f4c35be97bc0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b50b1} kind: Pod name: ensemble-graph-b50b1-97587847f-nkhzb namespace: kserve-ci-e2e-test resourceVersion: "22517" uid: 2c2e122f-76c2-49e6-8401-8af1f72d500f kind: Event lastTimestamp: "2026-06-15T06:13:11Z" message: 'Created container: ensemble-graph-b50b1' metadata: creationTimestamp: "2026-06-15T06:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:11Z" name: ensemble-graph-b50b1-97587847f-nkhzb.18b92c71040f04b3 namespace: kserve-ci-e2e-test resourceVersion: "22541" uid: 3ba258c8-a333-4716-833f-6bf143b41e96 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b50b1} kind: Pod name: ensemble-graph-b50b1-97587847f-nkhzb namespace: kserve-ci-e2e-test resourceVersion: "22517" uid: 2c2e122f-76c2-49e6-8401-8af1f72d500f kind: Event lastTimestamp: "2026-06-15T06:13:11Z" message: Started container ensemble-graph-b50b1 metadata: creationTimestamp: "2026-06-15T06:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:11Z" name: ensemble-graph-b50b1-97587847f-nkhzb.18b92c71048b8408 namespace: kserve-ci-e2e-test resourceVersion: "22542" uid: 4ef67e80-9bab-41df-b42a-11c3e7da15d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b50b1} kind: Pod name: ensemble-graph-b50b1-97587847f-nkhzb namespace: kserve-ci-e2e-test resourceVersion: "22517" uid: 2c2e122f-76c2-49e6-8401-8af1f72d500f kind: Event lastTimestamp: "2026-06-15T06:21:25Z" message: Stopping container ensemble-graph-b50b1 metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:25Z" name: ensemble-graph-b50b1-97587847f-nkhzb.18b92ce41474d7bd namespace: kserve-ci-e2e-test resourceVersion: "26341" uid: 7d0365d6-0da8-4c0b-8010-76b0423d3f55 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b50b1} kind: Pod name: ensemble-graph-b50b1-97587847f-nkhzb namespace: kserve-ci-e2e-test resourceVersion: "22517" uid: 2c2e122f-76c2-49e6-8401-8af1f72d500f kind: Event lastTimestamp: "2026-06-15T06:21:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:52Z" name: ensemble-graph-b50b1-97587847f-nkhzb.18b92ce491de6d78 namespace: kserve-ci-e2e-test resourceVersion: "26671" uid: ee046de6-4881-4e96-a8f3-94b67e5d6ec0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-b50b1-97587847f namespace: kserve-ci-e2e-test resourceVersion: "22511" uid: 07faa511-cf17-4a56-9a6b-dff24fa353e6 kind: Event lastTimestamp: "2026-06-15T06:13:10Z" message: 'Created pod: ensemble-graph-b50b1-97587847f-nkhzb' metadata: creationTimestamp: "2026-06-15T06:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:10Z" name: ensemble-graph-b50b1-97587847f.18b92c70e524572b namespace: kserve-ci-e2e-test resourceVersion: "22516" uid: fef554a7-d2ff-4f70-ae3b-22715c524cb0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "22508" uid: 7a7f5ed2-1297-4eea-8ee0-b5077fad92d3 kind: Event lastTimestamp: "2026-06-15T06:13:10Z" message: Scaled up replica set ensemble-graph-b50b1-97587847f from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:10Z" name: ensemble-graph-b50b1.18b92c70e4610cf4 namespace: kserve-ci-e2e-test resourceVersion: "22512" uid: 812c0e08-ae39-4451-9f19-fadfc42d0c9a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: 4a989f96-d076-4332-8721-a85d212f4c8e kind: Event lastTimestamp: "2026-06-15T06:13:11Z" message: InferenceGraph [ensemble-graph-b50b1] is Ready metadata: creationTimestamp: "2026-06-15T06:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:13:11Z" name: ensemble-graph-b50b1.18b92c70e96a1993 namespace: kserve-ci-e2e-test resourceVersion: "22530" uid: 0ad18c92-4e60-490c-a6c0-37f06d878605 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "22509" uid: bd679335-0546-42ec-8b84-7185a30662cd kind: Event lastTimestamp: "2026-06-15T06:13:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:26Z" name: ensemble-graph-b50b1.18b92c74637516e6 namespace: kserve-ci-e2e-test resourceVersion: "22671" uid: 836d5812-b045-4c57-b8e6-b719d56d0666 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "22509" uid: bd679335-0546-42ec-8b84-7185a30662cd kind: Event lastTimestamp: "2026-06-15T06:13:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:26Z" name: ensemble-graph-b50b1.18b92c7463c3c0be namespace: kserve-ci-e2e-test resourceVersion: "22672" uid: 8a146252-7d8c-4dba-90fb-5ed3761c7afc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "22669" uid: bd679335-0546-42ec-8b84-7185a30662cd kind: Event lastTimestamp: "2026-06-15T06:13:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:41Z" name: ensemble-graph-b50b1.18b92c77e27e25bc namespace: kserve-ci-e2e-test resourceVersion: "22784" uid: 228b258f-2b7d-4599-8b15-0266d3a0986f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "22669" uid: bd679335-0546-42ec-8b84-7185a30662cd kind: Event lastTimestamp: "2026-06-15T06:13:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:41Z" name: ensemble-graph-b50b1.18b92c77e2e8a477 namespace: kserve-ci-e2e-test resourceVersion: "22786" uid: e2eb09fe-d294-424f-92a0-cb21f4d59197 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:02:41.614060Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 namespace: kserve-ci-e2e-test resourceVersion: "17147" uid: 71312792-836b-4f3d-85e2-d3ee2e19fd55 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 to ip-10-0-129-86.ec2.internal metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:02:41Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2.18b92bde5b2bb295 namespace: kserve-ci-e2e-test resourceVersion: "17157" uid: 24b7c24b-9387-49cc-875f-dce007dea1b7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 71312792-836b-4f3d-85e2-d3ee2e19fd55 kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-0ac5a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:41Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2.18b92bde691d47f7 namespace: kserve-ci-e2e-test resourceVersion: "17178" uid: 80651659-b61a-4392-8e97-aec12b750068 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 namespace: kserve-ci-e2e-test resourceVersion: "17172" uid: 71312792-836b-4f3d-85e2-d3ee2e19fd55 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:02:42Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2.18b92bde98d2d4eb namespace: kserve-ci-e2e-test resourceVersion: "17201" uid: c6c3353c-1d71-466e-9bbc-861500d4fec7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 71312792-836b-4f3d-85e2-d3ee2e19fd55 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2.18b92bde9a480311 namespace: kserve-ci-e2e-test resourceVersion: "17203" uid: c4e0b9bc-acd7-4935-9778-8941f7944f8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 71312792-836b-4f3d-85e2-d3ee2e19fd55 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2.18b92bde9d64c7b9 namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 6054a412-b09c-4e68-82cc-d409bb3cde34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 71312792-836b-4f3d-85e2-d3ee2e19fd55 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2.18b92bde9df01a23 namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: 65692dd5-078b-4df0-bb66-001c07766460 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 71312792-836b-4f3d-85e2-d3ee2e19fd55 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2.18b92bde9dfb7b41 namespace: kserve-ci-e2e-test resourceVersion: "17207" uid: 00314997-6629-4704-aa58-01f18433451a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 71312792-836b-4f3d-85e2-d3ee2e19fd55 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2.18b92bdea09d50f6 namespace: kserve-ci-e2e-test resourceVersion: "17209" uid: e5959a35-0ea5-4144-9657-704f79040420 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 71312792-836b-4f3d-85e2-d3ee2e19fd55 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2.18b92bdea105f25a namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: 6407db3a-053b-4bef-8a82-e96173a41e5b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 71312792-836b-4f3d-85e2-d3ee2e19fd55 kind: Event lastTimestamp: "2026-06-15T06:11:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:58Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2.18b92bdea8c39ad7 namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: 0b65c18a-06ed-4de1-833c-b4a9d3eaff16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 71312792-836b-4f3d-85e2-d3ee2e19fd55 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:56Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2.18b92c5f8d749261 namespace: kserve-ci-e2e-test resourceVersion: "21453" uid: 42085377-c0be-4e7f-8274-1f086ab98c7f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 71312792-836b-4f3d-85e2-d3ee2e19fd55 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:56Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2.18b92c5f8d751df7 namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: c3964061-8440-4ce4-803e-04930c19faea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 71312792-836b-4f3d-85e2-d3ee2e19fd55 kind: Event lastTimestamp: "2026-06-15T06:11:58Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:58Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2.18b92c601d2a505a namespace: kserve-ci-e2e-test resourceVersion: "21505" uid: 06f859bc-983f-4035-8aae-fad846f6cb3b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0ac5a-predictor-64c5f5c9bf namespace: kserve-ci-e2e-test resourceVersion: "17140" uid: a90d7909-222a-4942-b825-14e57ad29019 kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: 'Created pod: error-404-isvc-0ac5a-predictor-64c5f5c9bf-lfns2' metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:41Z" name: error-404-isvc-0ac5a-predictor-64c5f5c9bf.18b92bde5aa06743 namespace: kserve-ci-e2e-test resourceVersion: "17149" uid: c04f72bc-c81a-4d54-a797-3ac8080df363 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17135" uid: c94d969d-8fcf-4389-b7c2-ec2361bfaf60 kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: Scaled up replica set error-404-isvc-0ac5a-predictor-64c5f5c9bf from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:41Z" name: error-404-isvc-0ac5a-predictor.18b92bde58981c34 namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: fb1e8088-7ccf-4222-92bc-e83fd010dfbe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:02:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17138" uid: d4951ee0-8fac-4976-8000-d35a4765baaa kind: Event lastTimestamp: "2026-06-15T06:03:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:02:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:11Z" name: error-404-isvc-0ac5a-predictor.18b92be1d7327c7e namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 52230216-9b48-4ede-89a3-62b87e7bbd85 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:02:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17138" uid: d4951ee0-8fac-4976-8000-d35a4765baaa kind: Event lastTimestamp: "2026-06-15T06:03:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:02:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:11Z" name: error-404-isvc-0ac5a-predictor.18b92be1d7a729e9 namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 45358b78-2eb3-43e1-89dc-1e8688c31688 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:03:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17388" uid: d4951ee0-8fac-4976-8000-d35a4765baaa kind: Event lastTimestamp: "2026-06-15T06:03:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:03:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:56Z" name: error-404-isvc-0ac5a-predictor.18b92be8d57e72f4 namespace: kserve-ci-e2e-test resourceVersion: "17891" uid: 4d9f36dc-ab61-4c0d-bb2e-d62ec814c33a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:03:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17388" uid: d4951ee0-8fac-4976-8000-d35a4765baaa kind: Event lastTimestamp: "2026-06-15T06:03:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:03:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:56Z" name: error-404-isvc-0ac5a-predictor.18b92be8d6456769 namespace: kserve-ci-e2e-test resourceVersion: "17892" uid: 6dcf2977-a0c5-4b6d-b2e4-7e69a25364a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0ac5a namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: b907bb2c-c03e-48a0-971b-5fcd56311e75 kind: Event lastTimestamp: "2026-06-15T06:03:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:03:29Z" name: error-404-isvc-0ac5a.18b92bde58801785 namespace: kserve-ci-e2e-test resourceVersion: "17652" uid: 0e1609c0-b8a0-4ecd-a7cb-5b00605cfbd4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0ac5a namespace: kserve-ci-e2e-test resourceVersion: "17145" uid: b907bb2c-c03e-48a0-971b-5fcd56311e75 kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-0ac5a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0ac5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:41Z" name: error-404-isvc-0ac5a.18b92bde634490ea namespace: kserve-ci-e2e-test resourceVersion: "17179" uid: 0356bfe0-3e11-4543-9c87-497b26a5d828 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0ac5a namespace: kserve-ci-e2e-test resourceVersion: "17145" uid: b907bb2c-c03e-48a0-971b-5fcd56311e75 kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0ac5a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:41Z" name: error-404-isvc-0ac5a.18b92bde6344e5b7 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: 5d782afd-b67e-48b6-83e0-e1ce0719f80a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0ac5a namespace: kserve-ci-e2e-test resourceVersion: "17649" uid: b907bb2c-c03e-48a0-971b-5fcd56311e75 kind: Event lastTimestamp: "2026-06-15T06:03:29Z" message: InferenceService [error-404-isvc-0ac5a] is Ready metadata: creationTimestamp: "2026-06-15T06:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:03:29Z" name: error-404-isvc-0ac5a.18b92be963aecbe0 namespace: kserve-ci-e2e-test resourceVersion: "17651" uid: bba39682-92d8-4bef-b516-98e929efee1f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:11:56.459882Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 6dce1cf9-097e-4686-8d0e-5fdef494f378 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-42a73-predictor-7b975775c9-6w5jz to ip-10-0-129-86.ec2.internal metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:11:56Z" name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz.18b92c5f8a8f0912 namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: 403cc990-9667-4bc2-b2d0-cc307aba6281 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 6dce1cf9-097e-4686-8d0e-5fdef494f378 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-42a73-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:56Z" name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz.18b92c5f9569219d namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: 8cd8e602-2501-421c-8335-883581a0d9ef reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: 6dce1cf9-097e-4686-8d0e-5fdef494f378 kind: Event lastTimestamp: "2026-06-15T06:11:57Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:11:57Z" name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz.18b92c5fc81b2970 namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: 2bafad00-d4ca-49a9-b287-053192ea7d99 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 6dce1cf9-097e-4686-8d0e-5fdef494f378 kind: Event lastTimestamp: "2026-06-15T06:11:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:57Z" name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz.18b92c5fc9a2178e namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: d2034791-ddd9-4716-a36e-e4276eed58e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 6dce1cf9-097e-4686-8d0e-5fdef494f378 kind: Event lastTimestamp: "2026-06-15T06:11:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:57Z" name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz.18b92c5fcc870baf namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: 272ac487-4dd1-4a9b-9078-f4368cbe1ad2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 6dce1cf9-097e-4686-8d0e-5fdef494f378 kind: Event lastTimestamp: "2026-06-15T06:11:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:57Z" name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz.18b92c5fccfbecab namespace: kserve-ci-e2e-test resourceVersion: "21490" uid: d25681cb-bd15-44b5-8c21-b900b9888624 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 6dce1cf9-097e-4686-8d0e-5fdef494f378 kind: Event lastTimestamp: "2026-06-15T06:11:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:57Z" name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz.18b92c5fcd062ef8 namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: 5f7c9455-beed-48b7-bffa-1bde86c2e08a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 6dce1cf9-097e-4686-8d0e-5fdef494f378 kind: Event lastTimestamp: "2026-06-15T06:11:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:57Z" name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz.18b92c5fd029a24c namespace: kserve-ci-e2e-test resourceVersion: "21492" uid: 56a849a4-855b-4770-91d2-5bec2ac291ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 6dce1cf9-097e-4686-8d0e-5fdef494f378 kind: Event lastTimestamp: "2026-06-15T06:11:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:57Z" name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz.18b92c5fd0a43860 namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: 5a55cdb3-c769-4c32-a32c-8d182271963e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:11:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 6dce1cf9-097e-4686-8d0e-5fdef494f378 kind: Event lastTimestamp: "2026-06-15T06:12:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:35Z" name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz.18b92c6024f318a4 namespace: kserve-ci-e2e-test resourceVersion: "22003" uid: 774e7aaf-0759-4bd2-b7f1-e9130fa322d6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 6dce1cf9-097e-4686-8d0e-5fdef494f378 kind: Event lastTimestamp: "2026-06-15T06:13:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:06Z" name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz.18b92c6fddce246c namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: 2e10663b-fee4-458f-bda2-8af7b0d7628d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 6dce1cf9-097e-4686-8d0e-5fdef494f378 kind: Event lastTimestamp: "2026-06-15T06:13:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:06Z" name: error-404-isvc-42a73-predictor-7b975775c9-6w5jz.18b92c6fddcf009d namespace: kserve-ci-e2e-test resourceVersion: "22363" uid: 35304e3f-2039-4da3-8ed6-94ba2606b62f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-42a73-predictor-7b975775c9 namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 61329715-3ac2-4d9c-afd9-f64a605e1b60 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: 'Created pod: error-404-isvc-42a73-predictor-7b975775c9-6w5jz' metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:56Z" name: error-404-isvc-42a73-predictor-7b975775c9.18b92c5f8a1474ad namespace: kserve-ci-e2e-test resourceVersion: "21430" uid: e4e7a520-ce30-4084-b942-692efbdafd85 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-42a73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: a93e955d-9d15-4c31-a919-e6fae9912dbd kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: Scaled up replica set error-404-isvc-42a73-predictor-7b975775c9 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:56Z" name: error-404-isvc-42a73-predictor.18b92c5f8863a6fa namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: ba35f176-962e-4997-9d90-3ab3736a0398 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-42a73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21421" uid: 0a024e79-9335-47f0-9615-a1daafe9a682 kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:11Z" name: error-404-isvc-42a73-predictor.18b92c6307bc0754 namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: 71c8f5cd-a82a-4ebe-b714-a1139fc4a796 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-42a73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21421" uid: 0a024e79-9335-47f0-9615-a1daafe9a682 kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:11Z" name: error-404-isvc-42a73-predictor.18b92c63091300e1 namespace: kserve-ci-e2e-test resourceVersion: "21779" uid: d73ee8e3-7224-4773-b048-a271b739fbc7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:12:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-42a73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21775" uid: 0a024e79-9335-47f0-9615-a1daafe9a682 kind: Event lastTimestamp: "2026-06-15T06:12:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:56Z" name: error-404-isvc-42a73-predictor.18b92c6687cda55c namespace: kserve-ci-e2e-test resourceVersion: "22220" uid: 462e6a47-757a-4d80-91f5-6279ba733088 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:12:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-42a73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21775" uid: 0a024e79-9335-47f0-9615-a1daafe9a682 kind: Event lastTimestamp: "2026-06-15T06:12:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:56Z" name: error-404-isvc-42a73-predictor.18b92c66881d9da5 namespace: kserve-ci-e2e-test resourceVersion: "22222" uid: 6673ca79-0f6a-420b-b224-c3057c7e4b6c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-42a73 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 9c81671b-84d7-4be1-ada3-bc2d3e36f26d kind: Event lastTimestamp: "2026-06-15T06:12:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:45Z" name: error-404-isvc-42a73.18b92c5f8820b415 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: ef8b49e0-80ba-4c76-8cc5-4aa993cc4af9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-42a73 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 9c81671b-84d7-4be1-ada3-bc2d3e36f26d kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-42a73": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-42a73": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:56Z" name: error-404-isvc-42a73.18b92c5f88786f0a namespace: kserve-ci-e2e-test resourceVersion: "21425" uid: 2e0337ce-6823-4efb-8966-63167fec6559 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-42a73 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 9c81671b-84d7-4be1-ada3-bc2d3e36f26d kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-42a73": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:56Z" name: error-404-isvc-42a73.18b92c5f8878ca3c namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: eab8e4ff-4f3e-4882-a2de-9a28535e3006 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-42a73 namespace: kserve-ci-e2e-test resourceVersion: "22099" uid: 9c81671b-84d7-4be1-ada3-bc2d3e36f26d kind: Event lastTimestamp: "2026-06-15T06:12:45Z" message: InferenceService [error-404-isvc-42a73] is Ready metadata: creationTimestamp: "2026-06-15T06:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:45Z" name: error-404-isvc-42a73.18b92c6adfed5a39 namespace: kserve-ci-e2e-test resourceVersion: "22101" uid: fbadc38b-7242-4355-b454-a1dac734d763 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:13:06.740030Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-44470-predictor-677b7d7455-tz4zt namespace: kserve-ci-e2e-test resourceVersion: "22415" uid: 5ef50367-92c0-4c69-9612-2a4829727c07 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-44470-predictor-677b7d7455-tz4zt to ip-10-0-129-86.ec2.internal metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:13:06Z" name: error-404-isvc-44470-predictor-677b7d7455-tz4zt.18b92c6fe7950509 namespace: kserve-ci-e2e-test resourceVersion: "22420" uid: 68601961-2e4d-4dc3-b4f1-56109f588b2e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-44470-predictor-677b7d7455-tz4zt namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 5ef50367-92c0-4c69-9612-2a4829727c07 kind: Event lastTimestamp: "2026-06-15T06:13:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-44470-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:06Z" name: error-404-isvc-44470-predictor-677b7d7455-tz4zt.18b92c6ff3afbfbe namespace: kserve-ci-e2e-test resourceVersion: "22429" uid: 56f5f4fb-6da3-497e-8c15-439ce05b4617 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-44470-predictor-677b7d7455-tz4zt namespace: kserve-ci-e2e-test resourceVersion: "22426" uid: 5ef50367-92c0-4c69-9612-2a4829727c07 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:13:07Z" name: error-404-isvc-44470-predictor-677b7d7455-tz4zt.18b92c702502196a namespace: kserve-ci-e2e-test resourceVersion: "22448" uid: 0b4983de-8daa-479d-90d0-b0bfa3645c0a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44470-predictor-677b7d7455-tz4zt namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 5ef50367-92c0-4c69-9612-2a4829727c07 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:07Z" name: error-404-isvc-44470-predictor-677b7d7455-tz4zt.18b92c702695d90c namespace: kserve-ci-e2e-test resourceVersion: "22452" uid: 62f74a77-5a5b-472d-bb18-4c9aba371f96 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44470-predictor-677b7d7455-tz4zt namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 5ef50367-92c0-4c69-9612-2a4829727c07 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:07Z" name: error-404-isvc-44470-predictor-677b7d7455-tz4zt.18b92c702a18d808 namespace: kserve-ci-e2e-test resourceVersion: "22455" uid: 3276601c-e60c-4f33-8482-b58cc8227914 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44470-predictor-677b7d7455-tz4zt namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 5ef50367-92c0-4c69-9612-2a4829727c07 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:07Z" name: error-404-isvc-44470-predictor-677b7d7455-tz4zt.18b92c702a91342f namespace: kserve-ci-e2e-test resourceVersion: "22456" uid: 91d8e651-6a1b-45c9-9840-ae96cc2beed8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44470-predictor-677b7d7455-tz4zt namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 5ef50367-92c0-4c69-9612-2a4829727c07 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:07Z" name: error-404-isvc-44470-predictor-677b7d7455-tz4zt.18b92c702a9bf669 namespace: kserve-ci-e2e-test resourceVersion: "22457" uid: 91fa3d86-2e31-42e2-b9e7-ddaef9706b7c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44470-predictor-677b7d7455-tz4zt namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 5ef50367-92c0-4c69-9612-2a4829727c07 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:07Z" name: error-404-isvc-44470-predictor-677b7d7455-tz4zt.18b92c702ef0c885 namespace: kserve-ci-e2e-test resourceVersion: "22459" uid: 61646361-b0ec-4a9a-8389-909a5115abf4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44470-predictor-677b7d7455-tz4zt namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 5ef50367-92c0-4c69-9612-2a4829727c07 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:07Z" name: error-404-isvc-44470-predictor-677b7d7455-tz4zt.18b92c702f69d911 namespace: kserve-ci-e2e-test resourceVersion: "22460" uid: 34ff39f6-78af-4754-ba17-28adb07fae1e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-06-15T06:13:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44470-predictor-677b7d7455-tz4zt namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 5ef50367-92c0-4c69-9612-2a4829727c07 kind: Event lastTimestamp: "2026-06-15T06:22:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:13:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:24Z" name: error-404-isvc-44470-predictor-677b7d7455-tz4zt.18b92c7046f6a6e3 namespace: kserve-ci-e2e-test resourceVersion: "27058" uid: 20c099af-9c6b-4b42-a96a-3087108bb273 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44470-predictor-677b7d7455-tz4zt namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 5ef50367-92c0-4c69-9612-2a4829727c07 kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:21Z" name: error-404-isvc-44470-predictor-677b7d7455-tz4zt.18b92cf11f6b1226 namespace: kserve-ci-e2e-test resourceVersion: "26995" uid: 42b6f771-0ce5-4790-b738-a11900471d22 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44470-predictor-677b7d7455-tz4zt namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 5ef50367-92c0-4c69-9612-2a4829727c07 kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:21Z" name: error-404-isvc-44470-predictor-677b7d7455-tz4zt.18b92cf11f6b99a5 namespace: kserve-ci-e2e-test resourceVersion: "26997" uid: b7285000-1607-4590-93f8-b37ce6f14620 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44470-predictor-677b7d7455-tz4zt namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 5ef50367-92c0-4c69-9612-2a4829727c07 kind: Event lastTimestamp: "2026-06-15T06:22:24Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:24Z" name: error-404-isvc-44470-predictor-677b7d7455-tz4zt.18b92cf1bb513cdb namespace: kserve-ci-e2e-test resourceVersion: "27057" uid: 07a1558e-1757-41e4-a43d-7b6a8be5d163 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-44470-predictor-677b7d7455 namespace: kserve-ci-e2e-test resourceVersion: "22408" uid: eddff63c-0158-49f3-b9ce-9a66b32480fd kind: Event lastTimestamp: "2026-06-15T06:13:06Z" message: 'Created pod: error-404-isvc-44470-predictor-677b7d7455-tz4zt' metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:06Z" name: error-404-isvc-44470-predictor-677b7d7455.18b92c6fe6fae324 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 3a799118-19e0-4dcb-b99a-e417f991ff2f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-44470-predictor namespace: kserve-ci-e2e-test resourceVersion: "22405" uid: 2ee270c6-914d-480f-8e19-b6e7bd0145e5 kind: Event lastTimestamp: "2026-06-15T06:13:06Z" message: Scaled up replica set error-404-isvc-44470-predictor-677b7d7455 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:06Z" name: error-404-isvc-44470-predictor.18b92c6fe57adddd namespace: kserve-ci-e2e-test resourceVersion: "22410" uid: 4df9f6f8-f0df-4974-b569-a5a60eed4f36 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44470-predictor namespace: kserve-ci-e2e-test resourceVersion: "22406" uid: d9af7ff6-67d3-4e3b-9209-8fc849ecf673 kind: Event lastTimestamp: "2026-06-15T06:13:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:21Z" name: error-404-isvc-44470-predictor.18b92c736434e749 namespace: kserve-ci-e2e-test resourceVersion: "22633" uid: 8f52ef93-16ff-4d06-82f7-7c218f672641 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44470-predictor namespace: kserve-ci-e2e-test resourceVersion: "22406" uid: d9af7ff6-67d3-4e3b-9209-8fc849ecf673 kind: Event lastTimestamp: "2026-06-15T06:13:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:21Z" name: error-404-isvc-44470-predictor.18b92c7364a530c9 namespace: kserve-ci-e2e-test resourceVersion: "22635" uid: 1137591b-33b2-4f97-883c-b9c6584bd4e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:13:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44470-predictor namespace: kserve-ci-e2e-test resourceVersion: "22634" uid: d9af7ff6-67d3-4e3b-9209-8fc849ecf673 kind: Event lastTimestamp: "2026-06-15T06:14:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:36Z" name: error-404-isvc-44470-predictor.18b92c76e370cd6a namespace: kserve-ci-e2e-test resourceVersion: "23250" uid: 3f5ca734-d5d2-4952-8be6-0a6f4d0e15e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:13:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44470-predictor namespace: kserve-ci-e2e-test resourceVersion: "22634" uid: d9af7ff6-67d3-4e3b-9209-8fc849ecf673 kind: Event lastTimestamp: "2026-06-15T06:14:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:36Z" name: error-404-isvc-44470-predictor.18b92c76e403e5ea namespace: kserve-ci-e2e-test resourceVersion: "23251" uid: 59a023dc-5cb6-4781-bacc-2d8042519be2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44470 namespace: kserve-ci-e2e-test resourceVersion: "22385" uid: 984a7d7d-d98d-4935-b06f-8c4afdf50e5d kind: Event lastTimestamp: "2026-06-15T06:13:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:13:54Z" name: error-404-isvc-44470.18b92c6fe54b8b8b namespace: kserve-ci-e2e-test resourceVersion: "22891" uid: 887dfc2f-806f-4d98-ac70-fd728af13f4b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44470 namespace: kserve-ci-e2e-test resourceVersion: "22385" uid: 984a7d7d-d98d-4935-b06f-8c4afdf50e5d kind: Event lastTimestamp: "2026-06-15T06:13:06Z" message: 'Failed to update status for InferenceService "error-404-isvc-44470": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-44470": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:13:06Z" name: error-404-isvc-44470.18b92c6fe5ad12ed namespace: kserve-ci-e2e-test resourceVersion: "22412" uid: 09323092-24de-49f1-a418-1f8978f6d22d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44470 namespace: kserve-ci-e2e-test resourceVersion: "22385" uid: 984a7d7d-d98d-4935-b06f-8c4afdf50e5d kind: Event lastTimestamp: "2026-06-15T06:13:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-44470": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:13:06Z" name: error-404-isvc-44470.18b92c6fe5ad772c namespace: kserve-ci-e2e-test resourceVersion: "22413" uid: ca8c1192-464c-483c-bb33-4be520c0ba72 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44470 namespace: kserve-ci-e2e-test resourceVersion: "22889" uid: 984a7d7d-d98d-4935-b06f-8c4afdf50e5d kind: Event lastTimestamp: "2026-06-15T06:13:54Z" message: InferenceService [error-404-isvc-44470] is Ready metadata: creationTimestamp: "2026-06-15T06:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:13:54Z" name: error-404-isvc-44470.18b92c7b01769b60 namespace: kserve-ci-e2e-test resourceVersion: "22890" uid: 7590e3f8-6de1-4c3d-a30f-9c402fcee481 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:21:25.938006Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 namespace: kserve-ci-e2e-test resourceVersion: "26418" uid: 384b72c2-64bc-4ba4-baba-46973675935b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 to ip-10-0-129-86.ec2.internal metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:21:25Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8.18b92ce42219ad30 namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 467bf985-f651-47c4-8f76-f77c71e6f4ec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 384b72c2-64bc-4ba4-baba-46973675935b kind: Event lastTimestamp: "2026-06-15T06:21:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-5c5b6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:26Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8.18b92ce42c10e048 namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: a720b28e-3aed-4855-8b82-6ab6500c02e7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 namespace: kserve-ci-e2e-test resourceVersion: "26429" uid: 384b72c2-64bc-4ba4-baba-46973675935b kind: Event lastTimestamp: "2026-06-15T06:21:26Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:21:26Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8.18b92ce45f49c52b namespace: kserve-ci-e2e-test resourceVersion: "26450" uid: d9828055-fa8a-45e9-88f8-706551d1a259 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 384b72c2-64bc-4ba4-baba-46973675935b kind: Event lastTimestamp: "2026-06-15T06:21:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:26Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8.18b92ce4609bd16e namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: cf45482e-04ab-42f2-84ec-a2301f03db53 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 384b72c2-64bc-4ba4-baba-46973675935b kind: Event lastTimestamp: "2026-06-15T06:21:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:27Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8.18b92ce4649f1b8d namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 8b876940-3063-4732-aba1-1d5ff1ccf12b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 384b72c2-64bc-4ba4-baba-46973675935b kind: Event lastTimestamp: "2026-06-15T06:21:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:27Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8.18b92ce46514d299 namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: a9604f8a-6d10-4a7b-80ad-a5aadd2c35a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 384b72c2-64bc-4ba4-baba-46973675935b kind: Event lastTimestamp: "2026-06-15T06:21:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:27Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8.18b92ce4651f9906 namespace: kserve-ci-e2e-test resourceVersion: "26459" uid: 1a82ac1e-0213-48b7-85d4-da00e0e13e32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 384b72c2-64bc-4ba4-baba-46973675935b kind: Event lastTimestamp: "2026-06-15T06:21:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:27Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8.18b92ce4687c68a9 namespace: kserve-ci-e2e-test resourceVersion: "26463" uid: 82c3733d-baba-469a-bde9-dc8ccc4c291d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 384b72c2-64bc-4ba4-baba-46973675935b kind: Event lastTimestamp: "2026-06-15T06:21:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:27Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8.18b92ce46940a512 namespace: kserve-ci-e2e-test resourceVersion: "26464" uid: 338113dd-53a9-41ff-805e-2b2cfb16571c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:21:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 384b72c2-64bc-4ba4-baba-46973675935b kind: Event lastTimestamp: "2026-06-15T06:22:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:21:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:04Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8.18b92ce4ad7fcde7 namespace: kserve-ci-e2e-test resourceVersion: "26761" uid: 0d772a48-c51c-40f3-9a1c-cd808fc8ec29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 384b72c2-64bc-4ba4-baba-46973675935b kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:36Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8.18b92cf47ed55657 namespace: kserve-ci-e2e-test resourceVersion: "27291" uid: 4ae94525-1055-4a89-abed-a7044f535dd6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 384b72c2-64bc-4ba4-baba-46973675935b kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:36Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8.18b92cf47ed5edb5 namespace: kserve-ci-e2e-test resourceVersion: "27293" uid: 6014a91f-351f-46d1-b325-5dfd879597c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8 namespace: kserve-ci-e2e-test resourceVersion: "26420" uid: 384b72c2-64bc-4ba4-baba-46973675935b kind: Event lastTimestamp: "2026-06-15T06:22:39Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:39Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8.18b92cf535baf37e namespace: kserve-ci-e2e-test resourceVersion: "27372" uid: 2b6a36d9-763f-494b-b02e-bfc1aa781496 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5c5b6-predictor-6b459ddd67 namespace: kserve-ci-e2e-test resourceVersion: "26407" uid: 41de0c3b-5291-42ac-a18e-6f0aeabe0fc9 kind: Event lastTimestamp: "2026-06-15T06:21:25Z" message: 'Created pod: error-404-isvc-5c5b6-predictor-6b459ddd67-6djb8' metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:21:25Z" name: error-404-isvc-5c5b6-predictor-6b459ddd67.18b92ce421b00118 namespace: kserve-ci-e2e-test resourceVersion: "26419" uid: 5569a5e2-3b48-4b9e-af66-ead6ee674fe9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5c5b6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26402" uid: 54ba243a-433d-46a8-8f68-eaa365cb5670 kind: Event lastTimestamp: "2026-06-15T06:21:25Z" message: Scaled up replica set error-404-isvc-5c5b6-predictor-6b459ddd67 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:21:25Z" name: error-404-isvc-5c5b6-predictor.18b92ce41ff58b5c namespace: kserve-ci-e2e-test resourceVersion: "26411" uid: 1fbf3897-d07f-4a01-a8c1-de634fbfd0aa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:21:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5c5b6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: be0d1d62-7992-4edb-aa46-d270eb675bde kind: Event lastTimestamp: "2026-06-15T06:21:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:21:55Z" name: error-404-isvc-5c5b6-predictor.18b92ce79f2a7a91 namespace: kserve-ci-e2e-test resourceVersion: "26701" uid: dfbe4fda-a1ce-4e68-bcd0-94489270969b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:21:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5c5b6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: be0d1d62-7992-4edb-aa46-d270eb675bde kind: Event lastTimestamp: "2026-06-15T06:21:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:21:55Z" name: error-404-isvc-5c5b6-predictor.18b92ce79f975dd8 namespace: kserve-ci-e2e-test resourceVersion: "26702" uid: beacd871-7ba7-45c3-a936-36c97e26049c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:22:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5c5b6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: be0d1d62-7992-4edb-aa46-d270eb675bde kind: Event lastTimestamp: "2026-06-15T06:22:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:25Z" name: error-404-isvc-5c5b6-predictor.18b92cee9d20121a namespace: kserve-ci-e2e-test resourceVersion: "27106" uid: f6534e3f-5599-44dd-8ddf-471185ae9424 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:22:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5c5b6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: be0d1d62-7992-4edb-aa46-d270eb675bde kind: Event lastTimestamp: "2026-06-15T06:22:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:25Z" name: error-404-isvc-5c5b6-predictor.18b92cee9d846650 namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: d902b0ac-3290-4147-bba2-3c38f3625005 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:21:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5c5b6 namespace: kserve-ci-e2e-test resourceVersion: "26387" uid: ce8be2a8-24f5-4cab-a6ce-0d323d0ce80a kind: Event lastTimestamp: "2026-06-15T06:22:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:22:14Z" name: error-404-isvc-5c5b6.18b92ce420757b4e namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 436a7ef1-6747-474c-8037-1e840a25269c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5c5b6 namespace: kserve-ci-e2e-test resourceVersion: "26387" uid: ce8be2a8-24f5-4cab-a6ce-0d323d0ce80a kind: Event lastTimestamp: "2026-06-15T06:21:25Z" message: 'Failed to update status for InferenceService "error-404-isvc-5c5b6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5c5b6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:21:25Z" name: error-404-isvc-5c5b6.18b92ce420c65eed namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 9d4b9803-81e1-4e8b-ad04-a32ba0e16ab2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5c5b6 namespace: kserve-ci-e2e-test resourceVersion: "26387" uid: ce8be2a8-24f5-4cab-a6ce-0d323d0ce80a kind: Event lastTimestamp: "2026-06-15T06:21:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5c5b6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:21:25Z" name: error-404-isvc-5c5b6.18b92ce420c6a90a namespace: kserve-ci-e2e-test resourceVersion: "26417" uid: 8bd789bb-aec5-4fda-824d-eda7dbd5ec58 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5c5b6 namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: ce8be2a8-24f5-4cab-a6ce-0d323d0ce80a kind: Event lastTimestamp: "2026-06-15T06:22:14Z" message: InferenceService [error-404-isvc-5c5b6] is Ready metadata: creationTimestamp: "2026-06-15T06:22:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:22:14Z" name: error-404-isvc-5c5b6.18b92cef6aa49ebf namespace: kserve-ci-e2e-test resourceVersion: "26843" uid: ea907448-e586-4e91-a2e8-35fb0d07ead7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:00:12.482620Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: 1ce2a81e-0d68-49c4-a709-bdb5bffc8e42 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 to ip-10-0-129-86.ec2.internal metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:00:12Z" name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4.18b92bbba23e9a92 namespace: kserve-ci-e2e-test resourceVersion: "15464" uid: 9a6502e5-c95f-4254-af1d-1e2caa78e8b7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: 1ce2a81e-0d68-49c4-a709-bdb5bffc8e42 kind: Event lastTimestamp: "2026-06-15T06:00:12Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:00:12Z" name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4.18b92bbbbbecca58 namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: d00777a2-6459-4cc2-8af2-a472d7f10c74 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 1ce2a81e-0d68-49c4-a709-bdb5bffc8e42 kind: Event lastTimestamp: "2026-06-15T06:00:12Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1619" metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:12Z" name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4.18b92bbbbd338e45 namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: 62f65f67-0859-4465-bb5b-af8181e1aba5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 1ce2a81e-0d68-49c4-a709-bdb5bffc8e42 kind: Event lastTimestamp: "2026-06-15T06:00:25Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1619" in 12.929s (12.929s including waiting). Image size: 1334643821 bytes.' metadata: creationTimestamp: "2026-06-15T06:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:25Z" name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4.18b92bbebfdba59b namespace: kserve-ci-e2e-test resourceVersion: "15624" uid: 071ec17d-dcd4-43fe-aa96-dd12a0eb059a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 1ce2a81e-0d68-49c4-a709-bdb5bffc8e42 kind: Event lastTimestamp: "2026-06-15T06:00:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:25Z" name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4.18b92bbec366a38a namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 241cc1e1-6b55-4ec7-96e5-a6557d9a74ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 1ce2a81e-0d68-49c4-a709-bdb5bffc8e42 kind: Event lastTimestamp: "2026-06-15T06:00:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:25Z" name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4.18b92bbec3df399a namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: 9200e025-e954-414a-80a3-1ade34344e9a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 1ce2a81e-0d68-49c4-a709-bdb5bffc8e42 kind: Event lastTimestamp: "2026-06-15T06:00:25Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-06-15T06:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:25Z" name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4.18b92bbec3ecd056 namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: 91b99a04-6d2f-4d93-b3df-31a4ca025d2d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 1ce2a81e-0d68-49c4-a709-bdb5bffc8e42 kind: Event lastTimestamp: "2026-06-15T06:00:28Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.261s (2.261s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-06-15T06:00:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:28Z" name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4.18b92bbf4ab537dc namespace: kserve-ci-e2e-test resourceVersion: "15666" uid: e26ca409-c4d6-4dcd-94e4-c3b25d373b41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 1ce2a81e-0d68-49c4-a709-bdb5bffc8e42 kind: Event lastTimestamp: "2026-06-15T06:00:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:00:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:28Z" name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4.18b92bbf4e53fb50 namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: ce384cd6-3cba-45c6-a6bc-a28c084eb251 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 1ce2a81e-0d68-49c4-a709-bdb5bffc8e42 kind: Event lastTimestamp: "2026-06-15T06:00:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:00:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:28Z" name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4.18b92bbf4ecd71bb namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: a75f79bc-0714-4013-8da8-5f565d9618c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:00:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 1ce2a81e-0d68-49c4-a709-bdb5bffc8e42 kind: Event lastTimestamp: "2026-06-15T06:01:05Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:00:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:05Z" name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4.18b92bbf9072f5ae namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: 75a05df4-4ff4-4203-ba17-1e44ee80732f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 1ce2a81e-0d68-49c4-a709-bdb5bffc8e42 kind: Event lastTimestamp: "2026-06-15T06:01:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:45Z" name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4.18b92bd154d4e46b namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 6b3387ff-b854-4701-b6a6-e379bd23d29d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 1ce2a81e-0d68-49c4-a709-bdb5bffc8e42 kind: Event lastTimestamp: "2026-06-15T06:01:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:45Z" name: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4.18b92bd154d57971 namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 7e16877d-2489-454e-85aa-6128f9b397da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5cf27-predictor-594bc754f5 namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: f8841518-0d12-4445-a56d-dd7dd859e7ea kind: Event lastTimestamp: "2026-06-15T06:00:12Z" message: 'Created pod: error-404-isvc-5cf27-predictor-594bc754f5-k2pf4' metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:12Z" name: error-404-isvc-5cf27-predictor-594bc754f5.18b92bbba1c38c55 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 3aa28ea0-c035-4194-9e07-5904aebbf3f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5cf27-predictor namespace: kserve-ci-e2e-test resourceVersion: "15449" uid: d75a34f0-9594-4a59-8fa7-d1c5542cc97b kind: Event lastTimestamp: "2026-06-15T06:00:12Z" message: Scaled up replica set error-404-isvc-5cf27-predictor-594bc754f5 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:12Z" name: error-404-isvc-5cf27-predictor.18b92bbb99cd9604 namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 767baa9a-e1a4-4969-987d-7ba37b1d8874 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:00:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5cf27-predictor namespace: kserve-ci-e2e-test resourceVersion: "15450" uid: e641f532-23f1-40e1-a6d8-13a3047379d8 kind: Event lastTimestamp: "2026-06-15T06:00:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:00:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:57Z" name: error-404-isvc-5cf27-predictor.18b92bbf183b57f9 namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 6572b73d-ca14-4075-a659-81eb13287381 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:00:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5cf27-predictor namespace: kserve-ci-e2e-test resourceVersion: "15450" uid: e641f532-23f1-40e1-a6d8-13a3047379d8 kind: Event lastTimestamp: "2026-06-15T06:00:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:00:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:57Z" name: error-404-isvc-5cf27-predictor.18b92bbf18b1270f namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: b3865605-6822-4302-930d-5a08309f8d88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:01:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5cf27-predictor namespace: kserve-ci-e2e-test resourceVersion: "15660" uid: e641f532-23f1-40e1-a6d8-13a3047379d8 kind: Event lastTimestamp: "2026-06-15T06:01:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:01:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:42Z" name: error-404-isvc-5cf27-predictor.18b92bc99514ebb2 namespace: kserve-ci-e2e-test resourceVersion: "16365" uid: 174492f3-07c8-48bf-bc90-0d83eda2b562 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:01:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5cf27-predictor namespace: kserve-ci-e2e-test resourceVersion: "15660" uid: e641f532-23f1-40e1-a6d8-13a3047379d8 kind: Event lastTimestamp: "2026-06-15T06:01:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:01:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:42Z" name: error-404-isvc-5cf27-predictor.18b92bc9958d5bc4 namespace: kserve-ci-e2e-test resourceVersion: "16366" uid: 9376cc3e-dee1-4340-8639-0a21c4ecbd5f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:00:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5cf27 namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: ddc74e8f-6298-4420-adf0-55facbce3002 kind: Event lastTimestamp: "2026-06-15T06:01:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:15Z" name: error-404-isvc-5cf27.18b92bbb9985c939 namespace: kserve-ci-e2e-test resourceVersion: "16110" uid: 7ff71973-f83d-4982-982d-061e2062c747 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5cf27 namespace: kserve-ci-e2e-test resourceVersion: "16108" uid: ddc74e8f-6298-4420-adf0-55facbce3002 kind: Event lastTimestamp: "2026-06-15T06:01:15Z" message: InferenceService [error-404-isvc-5cf27] is Ready metadata: creationTimestamp: "2026-06-15T06:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:15Z" name: error-404-isvc-5cf27.18b92bca4be5967e namespace: kserve-ci-e2e-test resourceVersion: "16109" uid: d97768aa-d327-475b-8483-0e53390bce1e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:22:36.235279Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2 namespace: kserve-ci-e2e-test resourceVersion: "27292" uid: 1550a953-a029-4824-95cc-8bd781de6aff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2 to ip-10-0-129-86.ec2.internal metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:22:36Z" name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2.18b92cf48024eab6 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: 96289f3d-f27a-4a42-9769-b8bddedc2b36 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2 namespace: kserve-ci-e2e-test resourceVersion: "27296" uid: 1550a953-a029-4824-95cc-8bd781de6aff kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-5edeb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:36Z" name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2.18b92cf487f608b5 namespace: kserve-ci-e2e-test resourceVersion: "27307" uid: 203dd468-9b21-4f13-8a48-968d423390f0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2 namespace: kserve-ci-e2e-test resourceVersion: "27303" uid: 1550a953-a029-4824-95cc-8bd781de6aff kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:22:37Z" name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2.18b92cf4bdc484a0 namespace: kserve-ci-e2e-test resourceVersion: "27331" uid: f7d287b3-031b-4943-ae89-73101a0e6baa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2 namespace: kserve-ci-e2e-test resourceVersion: "27296" uid: 1550a953-a029-4824-95cc-8bd781de6aff kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:37Z" name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2.18b92cf4bf040c69 namespace: kserve-ci-e2e-test resourceVersion: "27335" uid: 48bcf31b-70e4-437a-9ff2-04bbcb4dbf2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2 namespace: kserve-ci-e2e-test resourceVersion: "27296" uid: 1550a953-a029-4824-95cc-8bd781de6aff kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:37Z" name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2.18b92cf4c20acbc2 namespace: kserve-ci-e2e-test resourceVersion: "27339" uid: db122515-287e-44e6-bbc9-83228a05f9bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2 namespace: kserve-ci-e2e-test resourceVersion: "27296" uid: 1550a953-a029-4824-95cc-8bd781de6aff kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:37Z" name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2.18b92cf4c283613f namespace: kserve-ci-e2e-test resourceVersion: "27340" uid: e26ec571-7435-4cb6-aa76-e92af46b73e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2 namespace: kserve-ci-e2e-test resourceVersion: "27296" uid: 1550a953-a029-4824-95cc-8bd781de6aff kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:37Z" name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2.18b92cf4c28f7ebd namespace: kserve-ci-e2e-test resourceVersion: "27341" uid: 4926717e-fae1-4c56-9f7e-dc55053a494b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2 namespace: kserve-ci-e2e-test resourceVersion: "27296" uid: 1550a953-a029-4824-95cc-8bd781de6aff kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:37Z" name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2.18b92cf4c5bf0874 namespace: kserve-ci-e2e-test resourceVersion: "27347" uid: ff289e91-87d3-49a9-aac3-4a440b458b5a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2 namespace: kserve-ci-e2e-test resourceVersion: "27296" uid: 1550a953-a029-4824-95cc-8bd781de6aff kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:37Z" name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2.18b92cf4c63945c3 namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: 18b02008-32e0-46f6-bc4e-b161a90dfbde reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2 namespace: kserve-ci-e2e-test resourceVersion: "27296" uid: 1550a953-a029-4824-95cc-8bd781de6aff kind: Event lastTimestamp: "2026-06-15T06:23:14Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:23:14Z" name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2.18b92cf50add44c3 namespace: kserve-ci-e2e-test resourceVersion: "27689" uid: da971ed1-316b-4d72-9635-f00bb5e94381 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:31:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2 namespace: kserve-ci-e2e-test resourceVersion: "27296" uid: 1550a953-a029-4824-95cc-8bd781de6aff kind: Event lastTimestamp: "2026-06-15T06:31:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:31:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:31:51Z" name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2.18b92d75ae778376 namespace: kserve-ci-e2e-test resourceVersion: "31424" uid: 26f715cb-1c90-404c-a91b-ba83e75f37f2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:31:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2 namespace: kserve-ci-e2e-test resourceVersion: "27296" uid: 1550a953-a029-4824-95cc-8bd781de6aff kind: Event lastTimestamp: "2026-06-15T06:31:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:31:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:31:51Z" name: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2.18b92d75ae7820ec namespace: kserve-ci-e2e-test resourceVersion: "31425" uid: 56f10bd0-6675-4d86-8a9a-62740e531950 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5edeb-predictor-66b854b5f9 namespace: kserve-ci-e2e-test resourceVersion: "27282" uid: d9f542cf-a036-44ed-b53b-3bc8e013877a kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: 'Created pod: error-404-isvc-5edeb-predictor-66b854b5f9-5dfb2' metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:36Z" name: error-404-isvc-5edeb-predictor-66b854b5f9.18b92cf47f9c6e28 namespace: kserve-ci-e2e-test resourceVersion: "27295" uid: c76b4cb1-1536-4850-95b5-39c5405e16de reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5edeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27275" uid: 955b29ab-4c61-4596-b436-b3329e5df927 kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: Scaled up replica set error-404-isvc-5edeb-predictor-66b854b5f9 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:36Z" name: error-404-isvc-5edeb-predictor.18b92cf47e0949c5 namespace: kserve-ci-e2e-test resourceVersion: "27285" uid: 07c24474-5242-40b1-8d0a-5ab3067640c0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:22:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5edeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27281" uid: c913c318-5051-4642-bbd6-d79f626e1c05 kind: Event lastTimestamp: "2026-06-15T06:23:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:21Z" name: error-404-isvc-5edeb-predictor.18b92cf7fc8c2dcc namespace: kserve-ci-e2e-test resourceVersion: "27739" uid: b80cab93-4fa9-418c-8f2f-2efa22c05d42 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:22:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5edeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27281" uid: c913c318-5051-4642-bbd6-d79f626e1c05 kind: Event lastTimestamp: "2026-06-15T06:23:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:21Z" name: error-404-isvc-5edeb-predictor.18b92cf7fcf4d7c8 namespace: kserve-ci-e2e-test resourceVersion: "27740" uid: bcc0b4d7-629a-4763-879f-05b11e3e4ee9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:23:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5edeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27476" uid: c913c318-5051-4642-bbd6-d79f626e1c05 kind: Event lastTimestamp: "2026-06-15T06:24:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:24:06Z" name: error-404-isvc-5edeb-predictor.18b92d027908ff29 namespace: kserve-ci-e2e-test resourceVersion: "28174" uid: f58bc847-840f-4cd4-baef-640ee7a3f43b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:23:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5edeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27476" uid: c913c318-5051-4642-bbd6-d79f626e1c05 kind: Event lastTimestamp: "2026-06-15T06:24:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:24:06Z" name: error-404-isvc-5edeb-predictor.18b92d02797719ad namespace: kserve-ci-e2e-test resourceVersion: "28175" uid: 48b855ad-cc29-4f84-a7d5-e5ebb2424ffd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5edeb namespace: kserve-ci-e2e-test resourceVersion: "27253" uid: a8cb4601-3d03-4c48-bed4-67d380987233 kind: Event lastTimestamp: "2026-06-15T06:23:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:23:24Z" name: error-404-isvc-5edeb.18b92cf47ddbea84 namespace: kserve-ci-e2e-test resourceVersion: "27824" uid: ccdcad8c-3803-4d45-b006-bb650eb9e2fc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5edeb namespace: kserve-ci-e2e-test resourceVersion: "27253" uid: a8cb4601-3d03-4c48-bed4-67d380987233 kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-5edeb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5edeb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:22:36Z" name: error-404-isvc-5edeb.18b92cf47e419c5d namespace: kserve-ci-e2e-test resourceVersion: "27288" uid: cbf706cc-3fda-460f-9689-97883360c724 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5edeb namespace: kserve-ci-e2e-test resourceVersion: "27253" uid: a8cb4601-3d03-4c48-bed4-67d380987233 kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5edeb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:22:36Z" name: error-404-isvc-5edeb.18b92cf47e42062b namespace: kserve-ci-e2e-test resourceVersion: "27289" uid: 522d2bd4-51eb-4c6a-8464-41424218227d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5edeb namespace: kserve-ci-e2e-test resourceVersion: "27822" uid: a8cb4601-3d03-4c48-bed4-67d380987233 kind: Event lastTimestamp: "2026-06-15T06:23:24Z" message: InferenceService [error-404-isvc-5edeb] is Ready metadata: creationTimestamp: "2026-06-15T06:23:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:23:24Z" name: error-404-isvc-5edeb.18b92cffc559afc0 namespace: kserve-ci-e2e-test resourceVersion: "27823" uid: ad0f9b62-5c4e-41a8-8b33-e5e2387586a2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:22:21.741062Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6c800-predictor-54776894c5-6zr7k namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: 5efa7977-0132-444b-8cd0-956adf38943c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6c800-predictor-54776894c5-6zr7k to ip-10-0-129-86.ec2.internal metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:22:21Z" name: error-404-isvc-6c800-predictor-54776894c5-6zr7k.18b92cf12038b700 namespace: kserve-ci-e2e-test resourceVersion: "27001" uid: 1218c515-2a81-4e7e-b3d6-9d6c1d52caab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6c800-predictor-54776894c5-6zr7k namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: 5efa7977-0132-444b-8cd0-956adf38943c kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6c800-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:21Z" name: error-404-isvc-6c800-predictor-54776894c5-6zr7k.18b92cf1275ab956 namespace: kserve-ci-e2e-test resourceVersion: "27010" uid: b766f4be-063e-4e76-8a94-5f638c6a3646 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6c800-predictor-54776894c5-6zr7k namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 5efa7977-0132-444b-8cd0-956adf38943c kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:22:22Z" name: error-404-isvc-6c800-predictor-54776894c5-6zr7k.18b92cf15d8e79e2 namespace: kserve-ci-e2e-test resourceVersion: "27027" uid: 9c899ad3-1854-4e14-aeb0-9eb22a548a0a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c800-predictor-54776894c5-6zr7k namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: 5efa7977-0132-444b-8cd0-956adf38943c kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:22Z" name: error-404-isvc-6c800-predictor-54776894c5-6zr7k.18b92cf15edc5a08 namespace: kserve-ci-e2e-test resourceVersion: "27030" uid: a208152f-c46a-4e9b-95ce-6f8d271560d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c800-predictor-54776894c5-6zr7k namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: 5efa7977-0132-444b-8cd0-956adf38943c kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:22Z" name: error-404-isvc-6c800-predictor-54776894c5-6zr7k.18b92cf162877890 namespace: kserve-ci-e2e-test resourceVersion: "27036" uid: 32875bad-37cd-40dc-abb5-d56e29e7e52a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c800-predictor-54776894c5-6zr7k namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: 5efa7977-0132-444b-8cd0-956adf38943c kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:22Z" name: error-404-isvc-6c800-predictor-54776894c5-6zr7k.18b92cf162faac69 namespace: kserve-ci-e2e-test resourceVersion: "27037" uid: 435dcd02-9619-4671-889d-609453feb56f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c800-predictor-54776894c5-6zr7k namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: 5efa7977-0132-444b-8cd0-956adf38943c kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:22Z" name: error-404-isvc-6c800-predictor-54776894c5-6zr7k.18b92cf16305f984 namespace: kserve-ci-e2e-test resourceVersion: "27038" uid: bd807d35-925a-4333-9e8a-e09a171f891c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c800-predictor-54776894c5-6zr7k namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: 5efa7977-0132-444b-8cd0-956adf38943c kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:22Z" name: error-404-isvc-6c800-predictor-54776894c5-6zr7k.18b92cf165e20421 namespace: kserve-ci-e2e-test resourceVersion: "27039" uid: d5de8156-3fe1-41c6-8755-fdd28162dd84 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c800-predictor-54776894c5-6zr7k namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: 5efa7977-0132-444b-8cd0-956adf38943c kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:22Z" name: error-404-isvc-6c800-predictor-54776894c5-6zr7k.18b92cf16669b695 namespace: kserve-ci-e2e-test resourceVersion: "27040" uid: 60e875a4-c857-4af4-85f7-94355decf446 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c800-predictor-54776894c5-6zr7k namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: 5efa7977-0132-444b-8cd0-956adf38943c kind: Event lastTimestamp: "2026-06-15T06:23:00Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:23:00Z" name: error-404-isvc-6c800-predictor-54776894c5-6zr7k.18b92cf1c4c10060 namespace: kserve-ci-e2e-test resourceVersion: "27556" uid: 37608273-1ec1-4c93-832d-9f2f790761d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:39:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c800-predictor-54776894c5-6zr7k namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: 5efa7977-0132-444b-8cd0-956adf38943c kind: Event lastTimestamp: "2026-06-15T06:39:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:39:41Z" name: error-404-isvc-6c800-predictor-54776894c5-6zr7k.18b92de32dbb8977 namespace: kserve-ci-e2e-test resourceVersion: "34673" uid: 921b27b1-2648-4012-9e91-31fa032ee484 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:39:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c800-predictor-54776894c5-6zr7k namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: 5efa7977-0132-444b-8cd0-956adf38943c kind: Event lastTimestamp: "2026-06-15T06:39:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:39:41Z" name: error-404-isvc-6c800-predictor-54776894c5-6zr7k.18b92de32dbcb825 namespace: kserve-ci-e2e-test resourceVersion: "34674" uid: cc763ec6-1e8b-43ca-8f7f-21a97f825038 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6c800-predictor-54776894c5 namespace: kserve-ci-e2e-test resourceVersion: "26980" uid: 7edf37d8-7fdf-4faa-8c5d-953bf55534bc kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: 'Created pod: error-404-isvc-6c800-predictor-54776894c5-6zr7k' metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:21Z" name: error-404-isvc-6c800-predictor-54776894c5.18b92cf11fbf4f4b namespace: kserve-ci-e2e-test resourceVersion: "26996" uid: cf801aa3-b6b5-4408-b99f-b5e19af784b0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6c800-predictor namespace: kserve-ci-e2e-test resourceVersion: "26972" uid: 42afe2f1-c06e-4219-bd97-f9fa8db67903 kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: Scaled up replica set error-404-isvc-6c800-predictor-54776894c5 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:21Z" name: error-404-isvc-6c800-predictor.18b92cf11e1f5ddc namespace: kserve-ci-e2e-test resourceVersion: "26984" uid: d3bc9ce0-6a9d-49df-826b-17abf91ee81a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c800-predictor namespace: kserve-ci-e2e-test resourceVersion: "26979" uid: b3051808-fac5-420c-b25e-07dfb1e6868f kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:36Z" name: error-404-isvc-6c800-predictor.18b92cf49c97ef6f namespace: kserve-ci-e2e-test resourceVersion: "27319" uid: 5c07d268-c5b4-47a7-927f-ddd5ff25b144 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c800-predictor namespace: kserve-ci-e2e-test resourceVersion: "26979" uid: b3051808-fac5-420c-b25e-07dfb1e6868f kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:36Z" name: error-404-isvc-6c800-predictor.18b92cf49d00fa11 namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 222c64ad-b394-4041-a720-2e9298877659 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:22:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c800-predictor namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: b3051808-fac5-420c-b25e-07dfb1e6868f kind: Event lastTimestamp: "2026-06-15T06:23:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:36Z" name: error-404-isvc-6c800-predictor.18b92cf81bbc6b64 namespace: kserve-ci-e2e-test resourceVersion: "27951" uid: 14bcf529-0796-45d6-94f3-6f811db508ff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:22:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c800-predictor namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: b3051808-fac5-420c-b25e-07dfb1e6868f kind: Event lastTimestamp: "2026-06-15T06:23:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:36Z" name: error-404-isvc-6c800-predictor.18b92cf81c52b471 namespace: kserve-ci-e2e-test resourceVersion: "27952" uid: 6ff835a6-e29e-4f67-a020-ba6232997ea6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c800 namespace: kserve-ci-e2e-test resourceVersion: "26956" uid: b6a42e76-09e0-4caa-b609-ea5ef9cd8393 kind: Event lastTimestamp: "2026-06-15T06:23:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:23:10Z" name: error-404-isvc-6c800.18b92cf11de69885 namespace: kserve-ci-e2e-test resourceVersion: "27660" uid: de624b82-dc69-4c7b-81b0-f11c9ab4554a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c800 namespace: kserve-ci-e2e-test resourceVersion: "26956" uid: b6a42e76-09e0-4caa-b609-ea5ef9cd8393 kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: 'Failed to update status for InferenceService "error-404-isvc-6c800": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6c800": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:22:21Z" name: error-404-isvc-6c800.18b92cf11e731d13 namespace: kserve-ci-e2e-test resourceVersion: "26990" uid: 932c453c-74a8-4b89-994e-c43cb027c493 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c800 namespace: kserve-ci-e2e-test resourceVersion: "26956" uid: b6a42e76-09e0-4caa-b609-ea5ef9cd8393 kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6c800": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:22:21Z" name: error-404-isvc-6c800.18b92cf11e738386 namespace: kserve-ci-e2e-test resourceVersion: "26991" uid: 4c696fcd-dfaa-45b2-92d1-36dc3e0130ac reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c800 namespace: kserve-ci-e2e-test resourceVersion: "27658" uid: b6a42e76-09e0-4caa-b609-ea5ef9cd8393 kind: Event lastTimestamp: "2026-06-15T06:23:10Z" message: InferenceService [error-404-isvc-6c800] is Ready metadata: creationTimestamp: "2026-06-15T06:23:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:23:10Z" name: error-404-isvc-6c800.18b92cfc7faa0363 namespace: kserve-ci-e2e-test resourceVersion: "27659" uid: ce1a3728-05dd-44b1-84e5-0d4ad09cc512 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:11:00.783682Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: 46869756-215b-41e3-8812-8ac102fbc07d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v to ip-10-0-129-86.ec2.internal metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:11:00Z" name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v.18b92c5293ffa868 namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 252ae832-80ef-4bea-b20d-b97f172be185 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 46869756-215b-41e3-8812-8ac102fbc07d kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-78ee8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v.18b92c52a2d591d7 namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: ae050d79-8a13-4151-808f-411cc1d704f6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 46869756-215b-41e3-8812-8ac102fbc07d kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:11:01Z" name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v.18b92c52d18eb331 namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: 6ebb4592-f1f3-470f-a5be-a44210eba510 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 46869756-215b-41e3-8812-8ac102fbc07d kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v.18b92c52d32da5ed namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: aa9074c8-c605-4242-901b-f9d80e032fdd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 46869756-215b-41e3-8812-8ac102fbc07d kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v.18b92c52d647f5b1 namespace: kserve-ci-e2e-test resourceVersion: "20900" uid: f21c6950-ff90-46dd-890d-3b4dd582aadc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 46869756-215b-41e3-8812-8ac102fbc07d kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v.18b92c52d6bd18e6 namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: 1ecd2a6a-c5fc-44d7-9316-17981e6c333f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 46869756-215b-41e3-8812-8ac102fbc07d kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v.18b92c52d6c81710 namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: 30bfbfe8-e566-4692-a9e5-f2b2ff888d3b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 46869756-215b-41e3-8812-8ac102fbc07d kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v.18b92c52da03cae3 namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 61c078df-fe96-4a62-8a63-5e7034d2e779 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 46869756-215b-41e3-8812-8ac102fbc07d kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v.18b92c52da95c342 namespace: kserve-ci-e2e-test resourceVersion: "20905" uid: 27ed6c34-fa25-4418-8d2e-eb4038dd7ec1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:11:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 46869756-215b-41e3-8812-8ac102fbc07d kind: Event lastTimestamp: "2026-06-15T06:11:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:11:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:39Z" name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v.18b92c53496a30b0 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 0bb4ac6c-5377-4d1e-880f-2be0e990779a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 46869756-215b-41e3-8812-8ac102fbc07d kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:11Z" name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v.18b92c62ef1d5b9e namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 8a8aa1e5-107b-41f3-8d8b-4784d15010f5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 46869756-215b-41e3-8812-8ac102fbc07d kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:11Z" name: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v.18b92c62ef1e3d79 namespace: kserve-ci-e2e-test resourceVersion: "21748" uid: 931ed8ac-0188-4eb3-a27a-5764b5ef41c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-78ee8-predictor-69ddbf844c namespace: kserve-ci-e2e-test resourceVersion: "20841" uid: 482585c5-d5ca-4dc7-b0f3-9cdac8201e77 kind: Event lastTimestamp: "2026-06-15T06:11:00Z" message: 'Created pod: error-404-isvc-78ee8-predictor-69ddbf844c-r8w7v' metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:00Z" name: error-404-isvc-78ee8-predictor-69ddbf844c.18b92c52937c0b42 namespace: kserve-ci-e2e-test resourceVersion: "20858" uid: 76a62f1f-d075-496c-8ba8-364ad43f1569 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-78ee8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: d36c2a2c-b360-45b7-9928-eabd7125994e kind: Event lastTimestamp: "2026-06-15T06:11:00Z" message: Scaled up replica set error-404-isvc-78ee8-predictor-69ddbf844c from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:00Z" name: error-404-isvc-78ee8-predictor.18b92c52915c8eac namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: aa387bad-269c-4c4a-8b9e-8c9702c576d2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78ee8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20840" uid: 63665346-8f46-40de-ba79-ad8aa1316706 kind: Event lastTimestamp: "2026-06-15T06:11:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:30Z" name: error-404-isvc-78ee8-predictor.18b92c56100ef4b5 namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: da385fa7-835d-487b-b0dc-5e2cd5b75480 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78ee8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20840" uid: 63665346-8f46-40de-ba79-ad8aa1316706 kind: Event lastTimestamp: "2026-06-15T06:11:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:30Z" name: error-404-isvc-78ee8-predictor.18b92c56106c8f56 namespace: kserve-ci-e2e-test resourceVersion: "21150" uid: 07c9d0c3-0c37-4f30-a017-05849301c5ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78ee8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: 63665346-8f46-40de-ba79-ad8aa1316706 kind: Event lastTimestamp: "2026-06-15T06:12:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:00Z" name: error-404-isvc-78ee8-predictor.18b92c5d0e17782d namespace: kserve-ci-e2e-test resourceVersion: "21557" uid: 24fe79f6-4841-4e71-b995-6018883ed13e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:11:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78ee8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: 63665346-8f46-40de-ba79-ad8aa1316706 kind: Event lastTimestamp: "2026-06-15T06:12:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:00Z" name: error-404-isvc-78ee8-predictor.18b92c5d0e8f4d82 namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: ea55f63e-eddf-4e6a-8c04-005c66988e7d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78ee8 namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 781ac507-58cb-45b3-b011-c61894b1eb93 kind: Event lastTimestamp: "2026-06-15T06:11:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:49Z" name: error-404-isvc-78ee8.18b92c52914fce06 namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: e12d4b53-f163-4169-8fa4-d568c64e4fea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78ee8 namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 781ac507-58cb-45b3-b011-c61894b1eb93 kind: Event lastTimestamp: "2026-06-15T06:11:00Z" message: 'Failed to update status for InferenceService "error-404-isvc-78ee8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-78ee8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:00Z" name: error-404-isvc-78ee8.18b92c5291df6b16 namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: 3c83fe7c-159d-405d-84f4-16aca02680f0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78ee8 namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 781ac507-58cb-45b3-b011-c61894b1eb93 kind: Event lastTimestamp: "2026-06-15T06:11:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-78ee8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:00Z" name: error-404-isvc-78ee8.18b92c5291dfad5c namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: 5042ad49-76cb-4a67-b965-c5a7c8d8b7f1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78ee8 namespace: kserve-ci-e2e-test resourceVersion: "21305" uid: 781ac507-58cb-45b3-b011-c61894b1eb93 kind: Event lastTimestamp: "2026-06-15T06:11:49Z" message: InferenceService [error-404-isvc-78ee8] is Ready metadata: creationTimestamp: "2026-06-15T06:11:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:49Z" name: error-404-isvc-78ee8.18b92c5e045298be namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: f8c95111-cde4-495b-8757-f065f1d09137 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:12:11.046433Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: f9eb6f15-c14d-4b45-a14e-36d7b1275ff6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b50b1-predictor-7bfb854997-r62dl to ip-10-0-129-86.ec2.internal metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:12:11Z" name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl.18b92c62effc2d23 namespace: kserve-ci-e2e-test resourceVersion: "21753" uid: c9850d07-ef4d-4ff7-8620-324bd2ae7cfc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: f9eb6f15-c14d-4b45-a14e-36d7b1275ff6 kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-b50b1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:11Z" name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl.18b92c62fd3b8189 namespace: kserve-ci-e2e-test resourceVersion: "21763" uid: 2c214c47-f0f7-4773-8d5b-21d1a4213506 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: f9eb6f15-c14d-4b45-a14e-36d7b1275ff6 kind: Event lastTimestamp: "2026-06-15T06:12:12Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:12:12Z" name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl.18b92c632d5e057f namespace: kserve-ci-e2e-test resourceVersion: "21787" uid: 807ca812-ce60-4b02-8519-e1d87343529b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: f9eb6f15-c14d-4b45-a14e-36d7b1275ff6 kind: Event lastTimestamp: "2026-06-15T06:12:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:12Z" name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl.18b92c632eb0c0d3 namespace: kserve-ci-e2e-test resourceVersion: "21789" uid: a76c5d3d-16cb-43ac-b833-41359fbbca45 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: f9eb6f15-c14d-4b45-a14e-36d7b1275ff6 kind: Event lastTimestamp: "2026-06-15T06:12:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:12Z" name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl.18b92c6332469b59 namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: effe1edc-bd85-430b-ac8a-c1122bb16fa8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: f9eb6f15-c14d-4b45-a14e-36d7b1275ff6 kind: Event lastTimestamp: "2026-06-15T06:12:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:12Z" name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl.18b92c6332bb54dc namespace: kserve-ci-e2e-test resourceVersion: "21792" uid: 0a2f9f58-4b27-4174-8903-be0a16c4932c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: f9eb6f15-c14d-4b45-a14e-36d7b1275ff6 kind: Event lastTimestamp: "2026-06-15T06:12:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:12Z" name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl.18b92c6332c60af0 namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 23ac171f-611a-4929-8498-357a351262c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: f9eb6f15-c14d-4b45-a14e-36d7b1275ff6 kind: Event lastTimestamp: "2026-06-15T06:12:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:12Z" name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl.18b92c63357c5476 namespace: kserve-ci-e2e-test resourceVersion: "21794" uid: 9026a26d-0930-489a-ae1e-c30ff882d223 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: f9eb6f15-c14d-4b45-a14e-36d7b1275ff6 kind: Event lastTimestamp: "2026-06-15T06:12:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:12Z" name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl.18b92c6335f2a577 namespace: kserve-ci-e2e-test resourceVersion: "21795" uid: c31e59d3-1cb7-4ae2-bdd0-97f37a47fc57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:12:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: f9eb6f15-c14d-4b45-a14e-36d7b1275ff6 kind: Event lastTimestamp: "2026-06-15T06:12:50Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:50Z" name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl.18b92c63a711d13d namespace: kserve-ci-e2e-test resourceVersion: "22141" uid: 1575f72b-474d-4d3d-bd74-9e106c850d52 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: f9eb6f15-c14d-4b45-a14e-36d7b1275ff6 kind: Event lastTimestamp: "2026-06-15T06:21:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:25Z" name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl.18b92ce41d1113bd namespace: kserve-ci-e2e-test resourceVersion: "26389" uid: 27f2b6b5-ce77-4884-81d9-1496afac8752 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: f9eb6f15-c14d-4b45-a14e-36d7b1275ff6 kind: Event lastTimestamp: "2026-06-15T06:21:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:25Z" name: error-404-isvc-b50b1-predictor-7bfb854997-r62dl.18b92ce41d118b73 namespace: kserve-ci-e2e-test resourceVersion: "26391" uid: 20de7c93-4e05-4cc2-a41d-14e6397adf67 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b50b1-predictor-7bfb854997 namespace: kserve-ci-e2e-test resourceVersion: "21731" uid: 03c37a0c-deef-425e-9431-cea2a9124702 kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: 'Created pod: error-404-isvc-b50b1-predictor-7bfb854997-r62dl' metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:11Z" name: error-404-isvc-b50b1-predictor-7bfb854997.18b92c62ef8faea6 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: 06a960ef-8a68-478b-99d6-b451bfc609c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b50b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: 4e6b91cf-5607-48b4-ab12-115b9fa9b1ea kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: Scaled up replica set error-404-isvc-b50b1-predictor-7bfb854997 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:11Z" name: error-404-isvc-b50b1-predictor.18b92c62edb92253 namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 51300249-0e33-45e8-941e-19dc43df16e5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:12:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b50b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21730" uid: 632872ec-194b-471e-af9e-b93e2f8c04c6 kind: Event lastTimestamp: "2026-06-15T06:12:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:41Z" name: error-404-isvc-b50b1-predictor.18b92c666c94b4a2 namespace: kserve-ci-e2e-test resourceVersion: "22051" uid: 68d3fa59-8ea8-4bc9-8c12-536dfdef7b49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:12:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b50b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21730" uid: 632872ec-194b-471e-af9e-b93e2f8c04c6 kind: Event lastTimestamp: "2026-06-15T06:12:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:41Z" name: error-404-isvc-b50b1-predictor.18b92c666cebfca0 namespace: kserve-ci-e2e-test resourceVersion: "22052" uid: 804bb4a3-6567-4f76-8db6-1f64478cd619 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:12:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b50b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21926" uid: 632872ec-194b-471e-af9e-b93e2f8c04c6 kind: Event lastTimestamp: "2026-06-15T06:13:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:11Z" name: error-404-isvc-b50b1-predictor.18b92c6d6a72a721 namespace: kserve-ci-e2e-test resourceVersion: "22531" uid: 66d2179b-beb2-460e-95da-1f86399e8e21 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:12:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b50b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21926" uid: 632872ec-194b-471e-af9e-b93e2f8c04c6 kind: Event lastTimestamp: "2026-06-15T06:13:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:11Z" name: error-404-isvc-b50b1-predictor.18b92c6d6abfd222 namespace: kserve-ci-e2e-test resourceVersion: "22532" uid: 057e083f-d151-4e15-9128-b3102f18a221 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: 629ac2b2-ee3b-4e15-860b-8d3528bcccfd kind: Event lastTimestamp: "2026-06-15T06:13:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:13:00Z" name: error-404-isvc-b50b1.18b92c62ed949968 namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 8b404184-4614-4d96-aabe-a0128257f715 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: 629ac2b2-ee3b-4e15-860b-8d3528bcccfd kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-b50b1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b50b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:11Z" name: error-404-isvc-b50b1.18b92c62ee64af04 namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 72bedc47-440b-409c-ae06-54433b341917 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: 629ac2b2-ee3b-4e15-860b-8d3528bcccfd kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b50b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:11Z" name: error-404-isvc-b50b1.18b92c62ee650848 namespace: kserve-ci-e2e-test resourceVersion: "21744" uid: 9bfac3bc-b7ee-4e08-bddf-c135af01c113 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "22274" uid: 629ac2b2-ee3b-4e15-860b-8d3528bcccfd kind: Event lastTimestamp: "2026-06-15T06:13:00Z" message: InferenceService [error-404-isvc-b50b1] is Ready metadata: creationTimestamp: "2026-06-15T06:13:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:13:00Z" name: error-404-isvc-b50b1.18b92c6e61c1cc7d namespace: kserve-ci-e2e-test resourceVersion: "22275" uid: 67c0ef57-aa05-451d-afa4-01cce7420bd0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:01:45.689266Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ea4bd-predictor-776b445677-bptlg namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 9417a4f0-50ea-4f07-ac8b-390b6b25ef9f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ea4bd-predictor-776b445677-bptlg to ip-10-0-129-86.ec2.internal metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:01:45Z" name: error-404-isvc-ea4bd-predictor-776b445677-bptlg.18b92bd155cb17c1 namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: e6e911e9-d508-4733-b433-0445b30eee26 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ea4bd-predictor-776b445677-bptlg namespace: kserve-ci-e2e-test resourceVersion: "16496" uid: 9417a4f0-50ea-4f07-ac8b-390b6b25ef9f kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:01:46Z" name: error-404-isvc-ea4bd-predictor-776b445677-bptlg.18b92bd16f7ed10f namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 08132d56-e551-4259-a61e-46a5cd5a7381 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ea4bd-predictor-776b445677-bptlg namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 9417a4f0-50ea-4f07-ac8b-390b6b25ef9f kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:46Z" name: error-404-isvc-ea4bd-predictor-776b445677-bptlg.18b92bd1711c353e namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: 2445a02d-7f3f-4a14-ae3a-47ca808e0961 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ea4bd-predictor-776b445677-bptlg namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 9417a4f0-50ea-4f07-ac8b-390b6b25ef9f kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:46Z" name: error-404-isvc-ea4bd-predictor-776b445677-bptlg.18b92bd173b76a82 namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: cfcef192-3d2b-43ba-8a3e-b2e9307b2d9e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ea4bd-predictor-776b445677-bptlg namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 9417a4f0-50ea-4f07-ac8b-390b6b25ef9f kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:46Z" name: error-404-isvc-ea4bd-predictor-776b445677-bptlg.18b92bd17417bacb namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 4278741c-196a-41e4-9d61-1044a71b06ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ea4bd-predictor-776b445677-bptlg namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 9417a4f0-50ea-4f07-ac8b-390b6b25ef9f kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:46Z" name: error-404-isvc-ea4bd-predictor-776b445677-bptlg.18b92bd17422e827 namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 1ad35a3c-dae9-40bc-985a-08f0f4bc6f69 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ea4bd-predictor-776b445677-bptlg namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 9417a4f0-50ea-4f07-ac8b-390b6b25ef9f kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:46Z" name: error-404-isvc-ea4bd-predictor-776b445677-bptlg.18b92bd177def541 namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 9775537b-e0b4-4ea8-aa95-5be41f0330aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ea4bd-predictor-776b445677-bptlg namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 9417a4f0-50ea-4f07-ac8b-390b6b25ef9f kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:46Z" name: error-404-isvc-ea4bd-predictor-776b445677-bptlg.18b92bd1783aa2bd namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: 5603cf7a-812c-4749-8ce0-22b6e74c5c1a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ea4bd-predictor-776b445677-bptlg namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 9417a4f0-50ea-4f07-ac8b-390b6b25ef9f kind: Event lastTimestamp: "2026-06-15T06:11:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:02Z" name: error-404-isvc-ea4bd-predictor-776b445677-bptlg.18b92bd190e61ed1 namespace: kserve-ci-e2e-test resourceVersion: "20918" uid: 6f7c5a33-6394-4941-9741-39eaddb7cecb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ea4bd-predictor-776b445677-bptlg namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 9417a4f0-50ea-4f07-ac8b-390b6b25ef9f kind: Event lastTimestamp: "2026-06-15T06:11:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:00Z" name: error-404-isvc-ea4bd-predictor-776b445677-bptlg.18b92c5292de9521 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 7388f91b-2400-4dd4-9f4f-6a0b5e2f4861 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ea4bd-predictor-776b445677-bptlg namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 9417a4f0-50ea-4f07-ac8b-390b6b25ef9f kind: Event lastTimestamp: "2026-06-15T06:11:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:00Z" name: error-404-isvc-ea4bd-predictor-776b445677-bptlg.18b92c5292df21b8 namespace: kserve-ci-e2e-test resourceVersion: "20857" uid: 7b4cd8ef-4abe-4d61-a73e-5dae99286c45 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ea4bd-predictor-776b445677-bptlg namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 9417a4f0-50ea-4f07-ac8b-390b6b25ef9f kind: Event lastTimestamp: "2026-06-15T06:11:02Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:02Z" name: error-404-isvc-ea4bd-predictor-776b445677-bptlg.18b92c53055a18db namespace: kserve-ci-e2e-test resourceVersion: "20917" uid: 1c228c44-b041-4155-bd57-624306d9f812 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ea4bd-predictor-776b445677 namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: aa9b961a-7e90-4cba-be9e-dcc20e496fa9 kind: Event lastTimestamp: "2026-06-15T06:01:45Z" message: 'Created pod: error-404-isvc-ea4bd-predictor-776b445677-bptlg' metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:45Z" name: error-404-isvc-ea4bd-predictor-776b445677.18b92bd1556c6735 namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: 70fcfcea-5ce5-4dd6-97af-a3b8e70a4b46 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ea4bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: 170e7b00-975b-4c14-bc80-5ac36928eb9f kind: Event lastTimestamp: "2026-06-15T06:01:45Z" message: Scaled up replica set error-404-isvc-ea4bd-predictor-776b445677 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:45Z" name: error-404-isvc-ea4bd-predictor.18b92bd153687169 namespace: kserve-ci-e2e-test resourceVersion: "16475" uid: a9f96c31-d346-4ea3-8f54-62631342b83d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:02:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ea4bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: a0b575bb-ab1b-4ada-a93f-ddf5eb026977 kind: Event lastTimestamp: "2026-06-15T06:02:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:30Z" name: error-404-isvc-ea4bd-predictor.18b92bd4d22ff27b namespace: kserve-ci-e2e-test resourceVersion: "16913" uid: 8756b09b-ded4-4d34-8263-602a3bd8da40 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:02:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ea4bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: a0b575bb-ab1b-4ada-a93f-ddf5eb026977 kind: Event lastTimestamp: "2026-06-15T06:02:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:30Z" name: error-404-isvc-ea4bd-predictor.18b92bd4d2c0def6 namespace: kserve-ci-e2e-test resourceVersion: "16914" uid: 847bcc52-b21b-4f87-98b2-79a17ec05900 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:02:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ea4bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16665" uid: a0b575bb-ab1b-4ada-a93f-ddf5eb026977 kind: Event lastTimestamp: "2026-06-15T06:03:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:00Z" name: error-404-isvc-ea4bd-predictor.18b92bdf4f9f9caf namespace: kserve-ci-e2e-test resourceVersion: "17425" uid: 8009b3b5-9874-4f03-9f5d-ba294ac406c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:02:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ea4bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16665" uid: a0b575bb-ab1b-4ada-a93f-ddf5eb026977 kind: Event lastTimestamp: "2026-06-15T06:03:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:00Z" name: error-404-isvc-ea4bd-predictor.18b92bdf504e3917 namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: cb0dc43a-2ead-4da2-b894-6ab2bb74c62b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 357f2e24-890f-44ac-ae6b-8511e6db53c0 kind: Event lastTimestamp: "2026-06-15T06:02:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:32Z" name: error-404-isvc-ea4bd.18b92bd1536896a9 namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: bfaae5fe-7be0-4728-9718-83915b222467 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 357f2e24-890f-44ac-ae6b-8511e6db53c0 kind: Event lastTimestamp: "2026-06-15T06:01:45Z" message: 'Failed to update status for InferenceService "error-404-isvc-ea4bd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ea4bd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:45Z" name: error-404-isvc-ea4bd.18b92bd1540f8766 namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: 270733cf-9046-4309-af2e-2b872ce4cd26 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 357f2e24-890f-44ac-ae6b-8511e6db53c0 kind: Event lastTimestamp: "2026-06-15T06:01:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ea4bd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:45Z" name: error-404-isvc-ea4bd.18b92bd1540fe9d4 namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: 23a9754d-2335-4a4a-bca4-3b65bfaa63b5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "16970" uid: 357f2e24-890f-44ac-ae6b-8511e6db53c0 kind: Event lastTimestamp: "2026-06-15T06:02:32Z" message: InferenceService [error-404-isvc-ea4bd] is Ready metadata: creationTimestamp: "2026-06-15T06:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:32Z" name: error-404-isvc-ea4bd.18b92bdc4c2180eb namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: 0d0c4152-ae42-4b6e-97b1-d97d48dae0ac reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:00:11.286159Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-844fcc5454-69j9q to ip-10-0-132-4.ec2.internal metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:00:11Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bbb5aeef84b namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: 750043b9-9c05-48ad-b5d9-c3a23f0a7974 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:00:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:11Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bbb6619c200 namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: a06812dd-6578-493a-b74a-8b20e01a2f07 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15349" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:00:13Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:00:13Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bbbcf05f3aa namespace: kserve-ci-e2e-test resourceVersion: "15497" uid: 04dd52b1-10bb-435c-b6ab-e206a1662e69 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:00:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T06:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:13Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bbbd095c5eb namespace: kserve-ci-e2e-test resourceVersion: "15499" uid: 6a3a2cac-484d-474e-b84e-1f2f7617da2a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:00:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:13Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bbbd3a7fddf namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 656b5ee1-6ff7-41ce-a5b8-e10a031f8cd1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:00:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:13Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bbbd422f133 namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: c5ac18bb-4df6-417b-b43e-05d3927688c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:00:18Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1619" metadata: creationTimestamp: "2026-06-15T06:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:18Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bbced441a11 namespace: kserve-ci-e2e-test resourceVersion: "15554" uid: 3cfdcb63-4004-4fae-ba33-f07d92d3d36c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:00:41Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1619" in 23.444s (23.444s including waiting). Image size: 1555617714 bytes.' metadata: creationTimestamp: "2026-06-15T06:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:41Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bc262a4f9cf namespace: kserve-ci-e2e-test resourceVersion: "15786" uid: e8c17623-f9d6-4200-b743-e1e5839547bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:00:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:41Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bc267b754b5 namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: 9b462c5a-511f-445b-98c9-caa608d27107 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:00:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:41Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bc268c8c813 namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: 2fff85b0-68cd-4ab6-b92b-c708a42168a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:00:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-06-15T06:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:41Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bc26b69bd51 namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: 9e8f605f-593d-4ca7-bafa-cb8da8e5c51e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:00:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.208s (3.208s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-06-15T06:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:44Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bc32aa9bf10 namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: 9a1986b2-6d8f-4fbc-87b3-00f8d65fd66c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:00:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:45Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bc3368e4155 namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: 98c420e3-c4e0-417e-93ee-3699211c6ef0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:00:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:45Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bc337e17c7a namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 2e938be6-35c0-4944-b61a-49898f069ec5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-06-15T06:00:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:00:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bc37c44145c namespace: kserve-ci-e2e-test resourceVersion: "17195" uid: 036d523a-ffda-4581-940c-9505efaa78eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:41Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bde560e6c5b namespace: kserve-ci-e2e-test resourceVersion: "17129" uid: 23be7964-2022-401b-9f49-43d9063a0f3f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:41Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bde560ee9ac namespace: kserve-ci-e2e-test resourceVersion: "17136" uid: 8520acf7-ece9-44e3-a0c2-39fe038a9d1a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: f0a14ab3-458f-4a41-9db1-56c56adf897c kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q.18b92bde7e9b7db1 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 12f8c56a-a0dd-44d0-8ea6-720a2f647fc8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-844fcc5454 namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: 7ec5b62e-5734-4cf8-af80-899ccff1ee46 kind: Event lastTimestamp: "2026-06-15T06:00:11Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-844fcc5454-69j9q' metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:11Z" name: isvc-sklearn-graph-1-predictor-844fcc5454.18b92bbb5a7bc499 namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: 33bec6b5-e7a6-45e6-ba63-e2dfad842257 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: f9a64518-c42a-49c8-996a-ce33e811012f kind: Event lastTimestamp: "2026-06-15T06:00:11Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-844fcc5454 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:11Z" name: isvc-sklearn-graph-1-predictor.18b92bbb55cc8d3b namespace: kserve-ci-e2e-test resourceVersion: "15332" uid: 0d9e598d-f643-4d77-971c-fabdc311750c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: 995ad051-8687-4b3a-ab88-916da83b23e7 kind: Event lastTimestamp: "2026-06-15T06:01:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:11Z" name: isvc-sklearn-graph-1-predictor.18b92bbedbaf70b4 namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 06aa595c-2684-474e-9cd3-fe7d7fc89a0f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: 995ad051-8687-4b3a-ab88-916da83b23e7 kind: Event lastTimestamp: "2026-06-15T06:01:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:11Z" name: isvc-sklearn-graph-1-predictor.18b92bbedc1dc581 namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: f7568921-4362-46d7-96d0-8a9774b24acd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:01:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 995ad051-8687-4b3a-ab88-916da83b23e7 kind: Event lastTimestamp: "2026-06-15T06:02:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:26Z" name: isvc-sklearn-graph-1-predictor.18b92bccd85f3c94 namespace: kserve-ci-e2e-test resourceVersion: "16881" uid: ffb9b61d-b547-4cd6-bb57-2114799b8bc4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:01:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 995ad051-8687-4b3a-ab88-916da83b23e7 kind: Event lastTimestamp: "2026-06-15T06:02:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:26Z" name: isvc-sklearn-graph-1-predictor.18b92bccd8cd6d8e namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 1cf60a84-398b-44f0-a57a-a3c46513e7e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: 995ad051-8687-4b3a-ab88-916da83b23e7 kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: deployments.apps "isvc-sklearn-graph-1-predictor" not found metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:41Z" name: isvc-sklearn-graph-1-predictor.18b92bde539fb383 namespace: kserve-ci-e2e-test resourceVersion: "17114" uid: 554e0e97-6986-47fc-9374-8f4495e4e439 reason: FailedGetScale reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: 13dfa120-f065-4cf9-b7a1-1315ad0c9171 kind: Event lastTimestamp: "2026-06-15T06:02:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:02Z" name: isvc-sklearn-graph-1.18b92bbb5c159cf5 namespace: kserve-ci-e2e-test resourceVersion: "16691" uid: 1326e125-8594-4caf-af61-60545172673d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: 13dfa120-f065-4cf9-b7a1-1315ad0c9171 kind: Event lastTimestamp: "2026-06-15T06:02:02Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-06-15T06:02:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:02Z" name: isvc-sklearn-graph-1.18b92bd5339d4e01 namespace: kserve-ci-e2e-test resourceVersion: "16690" uid: 25294b90-0e9e-4ce1-a8ad-e73e6e0cd541 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:00:12.176073Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h to ip-10-0-129-86.ec2.internal metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:00:12Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bbb8ff8fdf3 namespace: kserve-ci-e2e-test resourceVersion: "15439" uid: 03217853-9791-47f0-ba00-2e1bf18c331f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:00:12Z" message: Add eth0 [10.134.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:00:12Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bbba9ed0e90 namespace: kserve-ci-e2e-test resourceVersion: "15472" uid: c79f8c18-6871-45b9-a69d-da12b1189435 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:00:12Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:12Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bbbab840c4e namespace: kserve-ci-e2e-test resourceVersion: "15477" uid: 13d4ea6b-2500-4b90-b4b2-6f5b700c4b64 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:00:24Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" in 11.555s (11.555s including waiting). Image size: 299992506 bytes.' metadata: creationTimestamp: "2026-06-15T06:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:24Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bbe5c4a9595 namespace: kserve-ci-e2e-test resourceVersion: "15606" uid: b4651e40-cef6-4cf4-83a4-0812c3b8c109 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:00:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:25Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bbebe0e6c9c namespace: kserve-ci-e2e-test resourceVersion: "15621" uid: 4b4138ca-a8ac-47e5-bb32-89f3c5a7d7f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:00:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:25Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bbebe880424 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 6efb8a4d-bbf8-482e-a832-8da56c149ab0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:00:30Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1619" metadata: creationTimestamp: "2026-06-15T06:00:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:30Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bbfcc68ecc5 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 25fa2462-4d3f-4765-b494-f12dc00b73c2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:00:36Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1619" in 5.669s (5.669s including waiting). Image size: 1555617714 bytes.' metadata: creationTimestamp: "2026-06-15T06:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:36Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bc11e548bb7 namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: b253019d-8cdb-4061-98e8-b3bc43f785f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:00:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:36Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bc12164d338 namespace: kserve-ci-e2e-test resourceVersion: "15737" uid: 16534ac7-8512-414d-8fe0-71b8b5333646 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:00:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:36Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bc121dccb8b namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: 01da4f1a-b18a-415f-b894-a41edd9da91a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:00:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:36Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bc121f1cee1 namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: 8cec4735-ec8c-42bb-a7c1-914ba4afc770 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:00:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:36Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bc1258c785b namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: 150a05bb-ab18-4d2e-a003-9852f3840333 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:00:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:36Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bc126004052 namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: 956826d8-bd7c-432e-b794-cbbf3ec6fabf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-06-15T06:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:02:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:43Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bc16f99042a namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: c9b63b6e-b913-4ebf-8411-187865825be7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:41Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bde51b8ea49 namespace: kserve-ci-e2e-test resourceVersion: "17097" uid: a6635723-9593-48f9-8e6a-440794234a5f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:41Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bde51b9c6e0 namespace: kserve-ci-e2e-test resourceVersion: "17099" uid: 0382a302-a588-40c1-9855-84c48cf61661 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e3425ca6-4438-45c1-9f1f-dfb5d3a4acf6 kind: Event lastTimestamp: "2026-06-15T06:02:43Z" message: 'Readiness probe failed: Get "https://10.134.0.18:8643/healthz": dial tcp 10.134.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:43Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h.18b92bdec60106d2 namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 7c89cc1b-4065-4200-a068-89f0f5442fa9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-86.ec2.internal source: component: kubelet host: ip-10-0-129-86.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-9d88d7b4b namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 1c0775b8-b191-4866-8700-8f2eb4ce2c45 kind: Event lastTimestamp: "2026-06-15T06:00:12Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-9d88d7b4b-ncg6h' metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:12Z" name: isvc-sklearn-graph-2-predictor-9d88d7b4b.18b92bbb8f7f614f namespace: kserve-ci-e2e-test resourceVersion: "15436" uid: e2c18631-74c1-4d7b-838d-3582ee7faaf5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: abf3b143-c803-4faf-a330-c9335ebffd48 kind: Event lastTimestamp: "2026-06-15T06:00:11Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-9d88d7b4b from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:11Z" name: isvc-sklearn-graph-2-predictor.18b92bbb72d694c8 namespace: kserve-ci-e2e-test resourceVersion: "15415" uid: 9e48211d-e82f-4cf5-9ed7-406c144af3af reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 633a15fd-f0fd-446d-8e59-448345dc1f0c kind: Event lastTimestamp: "2026-06-15T06:00:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:41Z" name: isvc-sklearn-graph-2-predictor.18b92bbef1865ceb namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: 251c2d45-e2b0-4a04-881b-b183c8f53c71 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 633a15fd-f0fd-446d-8e59-448345dc1f0c kind: Event lastTimestamp: "2026-06-15T06:00:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:41Z" name: isvc-sklearn-graph-2-predictor.18b92bbef20238ae namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: 483231b5-f709-4e11-8475-8fad5a9732c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-06-15T06:00:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 633a15fd-f0fd-446d-8e59-448345dc1f0c kind: Event lastTimestamp: "2026-06-15T06:02:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:26Z" name: isvc-sklearn-graph-2-predictor.18b92bc5ef87b322 namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: 10367406-3020-48b7-81b9-8f9cc627b67a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-06-15T06:00:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 633a15fd-f0fd-446d-8e59-448345dc1f0c kind: Event lastTimestamp: "2026-06-15T06:02:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:26Z" name: isvc-sklearn-graph-2-predictor.18b92bc5eff16cf9 namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: a96d146e-e462-45b7-ae8a-8e26c918242b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: bbb58c0d-5a14-46fe-bb78-4b01f1c6cc3d kind: Event lastTimestamp: "2026-06-15T06:01:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:53Z" name: isvc-sklearn-graph-2.18b92bbb72b52e78 namespace: kserve-ci-e2e-test resourceVersion: "16601" uid: a0da9b4e-af31-4ff5-987a-867e82db7563 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: bbb58c0d-5a14-46fe-bb78-4b01f1c6cc3d kind: Event lastTimestamp: "2026-06-15T06:01:53Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-06-15T06:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:53Z" name: isvc-sklearn-graph-2.18b92bd3268b5ae6 namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: aeb36afe-3f0d-430e-af0b-0c14c0c905dd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:00:11.576037Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15395" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-xllkw to ip-10-0-132-4.ec2.internal metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:00:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bbb6c352465 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 1e2b06bf-9a0f-45e4-a056-0e1ade1a1212 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:00:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bbb782ba018 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: c1fe2ea1-d460-4ead-995e-dfab2f1df5d0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:00:12Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:00:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bbba9e08a29 namespace: kserve-ci-e2e-test resourceVersion: "15471" uid: 0dc9c01c-d343-402b-94b6-857a47bdb04e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:00:12Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ba8edcbfb3f9312d158be16483785d7654e60c7090f262c42214fd2b29effada" already present on machine metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bbbab473d25 namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: 159db18b-8564-4147-9453-757419547859 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:00:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bbbae7b2bfb namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: 955ed9eb-b93e-47d3-985d-83631329d2c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:00:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-15T06:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bbbaef55a75 namespace: kserve-ci-e2e-test resourceVersion: "15479" uid: f62942a3-0ddd-43c2-9516-7700a42c4ad6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:00:17Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-06-15T06:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bbcb155c1f0 namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: e86834c9-9fce-4cc8-9082-518cea40cf4d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:00:41Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 24.671s (24.671s including waiting). Image size: 1304984825 bytes.' metadata: creationTimestamp: "2026-06-15T06:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bc26fd95a7b namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 2b17b6bc-cc19-4de6-a290-66a57932ac5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:00:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bc275d87843 namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: ba319bba-dbc7-4890-9652-b6f31ef0a160 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:00:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bc276a036d1 namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: e43f556c-4efe-44df-b666-9c98e9c36aa5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:00:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-06-15T06:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bc276adf3a0 namespace: kserve-ci-e2e-test resourceVersion: "15799" uid: da428c40-bcd9-4efb-9f13-ffe0b4b6318b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:00:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.143s (3.143s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-06-15T06:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bc332095ea1 namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: b9b31384-3c56-42c4-9438-5ed6e8253d19 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:00:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bc337a603a7 namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: f7fb5478-87f7-49f6-a87f-7aa477de4b49 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:00:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bc33959f519 namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: e216658c-4b2f-4cb7-857b-ae93977f5a63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-06-15T06:00:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:00:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bc37c448b85 namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: e1e5251f-9c54-46d8-bf75-2783454b8269 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bde5d8502ba namespace: kserve-ci-e2e-test resourceVersion: "17173" uid: eca4b8cf-68fa-415c-8ba9-1db28edaf159 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bde5d868308 namespace: kserve-ci-e2e-test resourceVersion: "17174" uid: 1f0f1acc-84c7-4b95-af73-65f1ab5e9b2b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-xllkw namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 60899697-ea88-42f6-8467-1280c652e6c3 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: isvc-xgboost-graph-predictor-555fc69d56-xllkw.18b92bde7e9c5a90 namespace: kserve-ci-e2e-test resourceVersion: "17193" uid: ee070b35-e693-4cb2-9b53-a9dc70f6c1b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-4.ec2.internal source: component: kubelet host: ip-10-0-132-4.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: 40d0d6fa-41f2-47de-a569-0e1993ac5b08 kind: Event lastTimestamp: "2026-06-15T06:00:11Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-xllkw' metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:11Z" name: isvc-xgboost-graph-predictor-555fc69d56.18b92bbb6b9f15e7 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: c0a72b9c-5c64-49f1-8a91-8c441fcadfb8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15380" uid: 49a2e3d0-f62e-4237-81a4-3e4ab5e4dc69 kind: Event lastTimestamp: "2026-06-15T06:00:11Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:11Z" name: isvc-xgboost-graph-predictor.18b92bbb66242b60 namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: ee67b60b-59ea-41cd-8b82-9541c50e6117 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: c57385d7-2679-4221-939c-810f2c26c649 kind: Event lastTimestamp: "2026-06-15T06:00:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:56Z" name: isvc-xgboost-graph-predictor.18b92bbee4f5e73e namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: b20d70d6-dca9-4226-933e-fea354e90059 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: c57385d7-2679-4221-939c-810f2c26c649 kind: Event lastTimestamp: "2026-06-15T06:00:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:56Z" name: isvc-xgboost-graph-predictor.18b92bbee5600deb namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: 6c3fac07-38e5-43ac-97d7-0f748a5dc277 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:01:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: c57385d7-2679-4221-939c-810f2c26c649 kind: Event lastTimestamp: "2026-06-15T06:02:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:01:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:11Z" name: isvc-xgboost-graph-predictor.18b92bc961ae0870 namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 79020857-95e7-4435-bc07-3de3ba0e4472 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:01:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: c57385d7-2679-4221-939c-810f2c26c649 kind: Event lastTimestamp: "2026-06-15T06:02:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:01:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:11Z" name: isvc-xgboost-graph-predictor.18b92bc9622cefb7 namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: b01de126-89d2-40ac-a367-7289c6756245 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: 31c9bf41-b301-413d-a088-3d19f9ac5c0b kind: Event lastTimestamp: "2026-06-15T06:01:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:42Z" name: isvc-xgboost-graph.18b92bbb6637dca8 namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: 66108ab2-d78e-4bdb-806e-907d1b762387 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: 31c9bf41-b301-413d-a088-3d19f9ac5c0b kind: Event lastTimestamp: "2026-06-15T06:01:42Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-06-15T06:01:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:42Z" name: isvc-xgboost-graph.18b92bd08af580db namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: ef5cacfe-53c6-4db4-9064-fe51b0fbb00e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:02:31.219613Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-567c7f9fdc-vssgs namespace: kserve-ci-e2e-test resourceVersion: "16928" uid: a12e02e6-4b5d-499d-a8d1-3903df7bb8c8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-567c7f9fdc-vssgs to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:02:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:02:31Z" name: model-chainer-567c7f9fdc-vssgs.18b92bdbef9d09a5 namespace: kserve-ci-e2e-test resourceVersion: "16933" uid: 880a9970-7a28-4392-854a-df334cc819a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:31Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-567c7f9fdc-vssgs namespace: kserve-ci-e2e-test resourceVersion: "16930" uid: a12e02e6-4b5d-499d-a8d1-3903df7bb8c8 kind: Event lastTimestamp: "2026-06-15T06:02:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:02:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:31Z" name: model-chainer-567c7f9fdc-vssgs.18b92bdbfe64fc84 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 953258d4-3c46-426d-90f8-8d4ebe21855f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:32Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-567c7f9fdc-vssgs namespace: kserve-ci-e2e-test resourceVersion: "16937" uid: a12e02e6-4b5d-499d-a8d1-3903df7bb8c8 kind: Event lastTimestamp: "2026-06-15T06:02:32Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:02:32Z" name: model-chainer-567c7f9fdc-vssgs.18b92bdc2d517bf4 namespace: kserve-ci-e2e-test resourceVersion: "16954" uid: 54cef090-0d96-47ec-a434-044dab16aa91 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-567c7f9fdc-vssgs namespace: kserve-ci-e2e-test resourceVersion: "16930" uid: a12e02e6-4b5d-499d-a8d1-3903df7bb8c8 kind: Event lastTimestamp: "2026-06-15T06:02:32Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" already present on machine metadata: creationTimestamp: "2026-06-15T06:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:32Z" name: model-chainer-567c7f9fdc-vssgs.18b92bdc2e96fcaf namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: 18fdd1fd-d0f3-4118-ab3f-f246d051bc0b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-567c7f9fdc-vssgs namespace: kserve-ci-e2e-test resourceVersion: "16930" uid: a12e02e6-4b5d-499d-a8d1-3903df7bb8c8 kind: Event lastTimestamp: "2026-06-15T06:02:32Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-06-15T06:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:32Z" name: model-chainer-567c7f9fdc-vssgs.18b92bdc32618d6b namespace: kserve-ci-e2e-test resourceVersion: "16960" uid: f9cf6b77-48e1-432a-ad73-f159dd8b2e5a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-567c7f9fdc-vssgs namespace: kserve-ci-e2e-test resourceVersion: "16930" uid: a12e02e6-4b5d-499d-a8d1-3903df7bb8c8 kind: Event lastTimestamp: "2026-06-15T06:02:32Z" message: Started container model-chainer metadata: creationTimestamp: "2026-06-15T06:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:32Z" name: model-chainer-567c7f9fdc-vssgs.18b92bdc32d887c3 namespace: kserve-ci-e2e-test resourceVersion: "16961" uid: c5b5e673-83dc-4cdc-9cb8-f3b140ce86a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-567c7f9fdc-vssgs namespace: kserve-ci-e2e-test resourceVersion: "16930" uid: a12e02e6-4b5d-499d-a8d1-3903df7bb8c8 kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:41Z" name: model-chainer-567c7f9fdc-vssgs.18b92bde4a9206a4 namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: f46a4888-78fb-4fae-8466-b5da600569fd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:02:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-567c7f9fdc-vssgs namespace: kserve-ci-e2e-test resourceVersion: "16930" uid: a12e02e6-4b5d-499d-a8d1-3903df7bb8c8 kind: Event lastTimestamp: "2026-06-15T06:03:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:08Z" name: model-chainer-567c7f9fdc-vssgs.18b92bdee4a7e1ad namespace: kserve-ci-e2e-test resourceVersion: "17481" uid: 395766d8-e1a8-478a-a449-463f4e27a9ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-567c7f9fdc namespace: kserve-ci-e2e-test resourceVersion: "16925" uid: 3434f4b9-e7bc-4525-93bf-a5d34607415e kind: Event lastTimestamp: "2026-06-15T06:02:31Z" message: 'Created pod: model-chainer-567c7f9fdc-vssgs' metadata: creationTimestamp: "2026-06-15T06:02:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:31Z" name: model-chainer-567c7f9fdc.18b92bdbef16c1d3 namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: d45fdedc-889f-490e-bb90-fa06e982b41a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16923" uid: 38e3dd3c-81a1-4604-a05e-b8c9b3b3e89d kind: Event lastTimestamp: "2026-06-15T06:02:31Z" message: Scaled up replica set model-chainer-567c7f9fdc from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:02:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:31Z" name: model-chainer.18b92bdbee4b6759 namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 7c7c9d8e-c44c-436b-a668-9a26e1dd8d14 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: b7a32ac9-19fa-44aa-9575-f024bb243226 kind: Event lastTimestamp: "2026-06-15T06:02:31Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-06-15T06:02:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:31Z" name: model-chainer.18b92bdbf38a405e namespace: kserve-ci-e2e-test resourceVersion: "16941" uid: 3765493c-a075-4965-a50e-0337f580285a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:03:41.501486Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0ac5a-665fcb9dd4-hxvgp namespace: kserve-ci-e2e-test resourceVersion: "17749" uid: 6f98403b-2ccc-48a2-a20c-806943d57b04 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-0ac5a-665fcb9dd4-hxvgp to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:03:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:03:41Z" name: sequence-graph-0ac5a-665fcb9dd4-hxvgp.18b92bec4cbd558f namespace: kserve-ci-e2e-test resourceVersion: "17755" uid: e2fd46d0-4c98-4532-87b0-034a3f93c05d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:41Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0ac5a-665fcb9dd4-hxvgp namespace: kserve-ci-e2e-test resourceVersion: "17752" uid: 6f98403b-2ccc-48a2-a20c-806943d57b04 kind: Event lastTimestamp: "2026-06-15T06:03:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-0ac5a-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:41Z" name: sequence-graph-0ac5a-665fcb9dd4-hxvgp.18b92bec56b04b3a namespace: kserve-ci-e2e-test resourceVersion: "17773" uid: 9eb107ab-5c54-430c-bae3-e4baddefd093 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:42Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0ac5a-665fcb9dd4-hxvgp namespace: kserve-ci-e2e-test resourceVersion: "17762" uid: 6f98403b-2ccc-48a2-a20c-806943d57b04 kind: Event lastTimestamp: "2026-06-15T06:03:42Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:03:42Z" name: sequence-graph-0ac5a-665fcb9dd4-hxvgp.18b92bec8a842c19 namespace: kserve-ci-e2e-test resourceVersion: "17781" uid: 4c933d14-3b3d-4a6f-9fd7-e772b0f2d1db reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0ac5a} kind: Pod name: sequence-graph-0ac5a-665fcb9dd4-hxvgp namespace: kserve-ci-e2e-test resourceVersion: "17752" uid: 6f98403b-2ccc-48a2-a20c-806943d57b04 kind: Event lastTimestamp: "2026-06-15T06:03:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" already present on machine metadata: creationTimestamp: "2026-06-15T06:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:42Z" name: sequence-graph-0ac5a-665fcb9dd4-hxvgp.18b92bec8bc80184 namespace: kserve-ci-e2e-test resourceVersion: "17783" uid: 8eef248d-6177-471c-b513-0a826dbc1beb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0ac5a} kind: Pod name: sequence-graph-0ac5a-665fcb9dd4-hxvgp namespace: kserve-ci-e2e-test resourceVersion: "17752" uid: 6f98403b-2ccc-48a2-a20c-806943d57b04 kind: Event lastTimestamp: "2026-06-15T06:03:42Z" message: 'Created container: sequence-graph-0ac5a' metadata: creationTimestamp: "2026-06-15T06:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:42Z" name: sequence-graph-0ac5a-665fcb9dd4-hxvgp.18b92bec8ed0f5d4 namespace: kserve-ci-e2e-test resourceVersion: "17784" uid: 89088222-052c-4af6-9a50-2985e4a7fff2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0ac5a} kind: Pod name: sequence-graph-0ac5a-665fcb9dd4-hxvgp namespace: kserve-ci-e2e-test resourceVersion: "17752" uid: 6f98403b-2ccc-48a2-a20c-806943d57b04 kind: Event lastTimestamp: "2026-06-15T06:03:42Z" message: Started container sequence-graph-0ac5a metadata: creationTimestamp: "2026-06-15T06:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:03:42Z" name: sequence-graph-0ac5a-665fcb9dd4-hxvgp.18b92bec8f470afa namespace: kserve-ci-e2e-test resourceVersion: "17785" uid: 2030d319-ec14-4906-9925-3e5a04111f09 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0ac5a} kind: Pod name: sequence-graph-0ac5a-665fcb9dd4-hxvgp namespace: kserve-ci-e2e-test resourceVersion: "17752" uid: 6f98403b-2ccc-48a2-a20c-806943d57b04 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: Stopping container sequence-graph-0ac5a metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:56Z" name: sequence-graph-0ac5a-665fcb9dd4-hxvgp.18b92c5f7c917f3b namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: acf1034d-4fae-4032-b134-35dffd98c28d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:11:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0ac5a} kind: Pod name: sequence-graph-0ac5a-665fcb9dd4-hxvgp namespace: kserve-ci-e2e-test resourceVersion: "17752" uid: 6f98403b-2ccc-48a2-a20c-806943d57b04 kind: Event lastTimestamp: "2026-06-15T06:12:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:24Z" name: sequence-graph-0ac5a-665fcb9dd4-hxvgp.18b92c60306d9c84 namespace: kserve-ci-e2e-test resourceVersion: "21908" uid: 514eec01-e69d-41fd-810f-242340ac4e17 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-0ac5a-665fcb9dd4 namespace: kserve-ci-e2e-test resourceVersion: "17746" uid: 93967fe4-d4b8-466e-911e-e02878acd961 kind: Event lastTimestamp: "2026-06-15T06:03:41Z" message: 'Created pod: sequence-graph-0ac5a-665fcb9dd4-hxvgp' metadata: creationTimestamp: "2026-06-15T06:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:41Z" name: sequence-graph-0ac5a-665fcb9dd4.18b92bec4c4f4cd2 namespace: kserve-ci-e2e-test resourceVersion: "17751" uid: 60aaa29e-4478-4d47-9fa6-5ef6f890bcbf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-0ac5a namespace: kserve-ci-e2e-test resourceVersion: "17744" uid: 54132a2b-5aa2-4e19-bafe-0a69f610e4c8 kind: Event lastTimestamp: "2026-06-15T06:03:41Z" message: Scaled up replica set sequence-graph-0ac5a-665fcb9dd4 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:41Z" name: sequence-graph-0ac5a.18b92bec4b437ef1 namespace: kserve-ci-e2e-test resourceVersion: "17747" uid: 9ba6f74e-6581-425a-a39c-1fef9c4344d6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0ac5a namespace: kserve-ci-e2e-test resourceVersion: "17765" uid: e0df83a2-8f9b-4a34-840d-6b257ea23b7a kind: Event lastTimestamp: "2026-06-15T06:03:41Z" message: InferenceGraph [sequence-graph-0ac5a] is Ready metadata: creationTimestamp: "2026-06-15T06:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:03:41Z" name: sequence-graph-0ac5a.18b92bec50d0e090 namespace: kserve-ci-e2e-test resourceVersion: "17767" uid: adfaff4e-eeda-45d4-b59e-a59a8dd81c47 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:03:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0ac5a namespace: kserve-ci-e2e-test resourceVersion: "17745" uid: 50bab6c9-b3d5-47b0-8344-b3a07fe6c010 kind: Event lastTimestamp: "2026-06-15T06:04:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:03:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:04:11Z" name: sequence-graph-0ac5a.18b92befc9dcdb32 namespace: kserve-ci-e2e-test resourceVersion: "17991" uid: 8c8ba8c9-d0c2-4af6-8ad2-5dac535fc6c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:03:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0ac5a namespace: kserve-ci-e2e-test resourceVersion: "17745" uid: 50bab6c9-b3d5-47b0-8344-b3a07fe6c010 kind: Event lastTimestamp: "2026-06-15T06:04:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:03:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:04:11Z" name: sequence-graph-0ac5a.18b92befca49a764 namespace: kserve-ci-e2e-test resourceVersion: "17992" uid: 8c0ddb62-7bc0-4c46-804e-3841c60f0a72 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:12:56.400726Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-42a73-84c6b98785-vsxjq namespace: kserve-ci-e2e-test resourceVersion: "22199" uid: 4a6942c9-6b46-45b5-ab87-54bdb5428385 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-42a73-84c6b98785-vsxjq to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:12:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:12:56Z" name: sequence-graph-42a73-84c6b98785-vsxjq.18b92c6d7f4fc1ee namespace: kserve-ci-e2e-test resourceVersion: "22207" uid: 20734a0b-14a2-4c17-97af-29f6c23aa685 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:56Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-42a73-84c6b98785-vsxjq namespace: kserve-ci-e2e-test resourceVersion: "22213" uid: 4a6942c9-6b46-45b5-ab87-54bdb5428385 kind: Event lastTimestamp: "2026-06-15T06:12:56Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:12:56Z" name: sequence-graph-42a73-84c6b98785-vsxjq.18b92c6d9986736b namespace: kserve-ci-e2e-test resourceVersion: "22226" uid: 605da0b0-0784-4a25-93a5-62e22fd1c452 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-42a73} kind: Pod name: sequence-graph-42a73-84c6b98785-vsxjq namespace: kserve-ci-e2e-test resourceVersion: "22202" uid: 4a6942c9-6b46-45b5-ab87-54bdb5428385 kind: Event lastTimestamp: "2026-06-15T06:12:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" already present on machine metadata: creationTimestamp: "2026-06-15T06:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:56Z" name: sequence-graph-42a73-84c6b98785-vsxjq.18b92c6d9b06b131 namespace: kserve-ci-e2e-test resourceVersion: "22228" uid: 59bf9594-750b-4b2b-88a6-d4291b4aaf63 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-42a73} kind: Pod name: sequence-graph-42a73-84c6b98785-vsxjq namespace: kserve-ci-e2e-test resourceVersion: "22202" uid: 4a6942c9-6b46-45b5-ab87-54bdb5428385 kind: Event lastTimestamp: "2026-06-15T06:12:56Z" message: 'Created container: sequence-graph-42a73' metadata: creationTimestamp: "2026-06-15T06:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:56Z" name: sequence-graph-42a73-84c6b98785-vsxjq.18b92c6d9e27aaca namespace: kserve-ci-e2e-test resourceVersion: "22229" uid: 964842ac-4a7c-4169-8802-b8dddb3a2241 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-42a73} kind: Pod name: sequence-graph-42a73-84c6b98785-vsxjq namespace: kserve-ci-e2e-test resourceVersion: "22202" uid: 4a6942c9-6b46-45b5-ab87-54bdb5428385 kind: Event lastTimestamp: "2026-06-15T06:12:56Z" message: Started container sequence-graph-42a73 metadata: creationTimestamp: "2026-06-15T06:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:56Z" name: sequence-graph-42a73-84c6b98785-vsxjq.18b92c6d9ea3e4b1 namespace: kserve-ci-e2e-test resourceVersion: "22230" uid: eff0191c-9e72-4e38-8703-272d4be2240a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-42a73} kind: Pod name: sequence-graph-42a73-84c6b98785-vsxjq namespace: kserve-ci-e2e-test resourceVersion: "22202" uid: 4a6942c9-6b46-45b5-ab87-54bdb5428385 kind: Event lastTimestamp: "2026-06-15T06:13:06Z" message: Stopping container sequence-graph-42a73 metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:06Z" name: sequence-graph-42a73-84c6b98785-vsxjq.18b92c6fda218835 namespace: kserve-ci-e2e-test resourceVersion: "22338" uid: ada1e441-5a6e-4cca-9d3e-bb743ba52444 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:13:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-42a73} kind: Pod name: sequence-graph-42a73-84c6b98785-vsxjq namespace: kserve-ci-e2e-test resourceVersion: "22202" uid: 4a6942c9-6b46-45b5-ab87-54bdb5428385 kind: Event lastTimestamp: "2026-06-15T06:13:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:13:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:33Z" name: sequence-graph-42a73-84c6b98785-vsxjq.18b92c705e712672 namespace: kserve-ci-e2e-test resourceVersion: "22723" uid: ef4eb216-5bc0-41de-b593-b448e241feb5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-42a73-84c6b98785 namespace: kserve-ci-e2e-test resourceVersion: "22196" uid: a3c365dd-bc36-460a-9fb9-31b358273dc7 kind: Event lastTimestamp: "2026-06-15T06:12:56Z" message: 'Created pod: sequence-graph-42a73-84c6b98785-vsxjq' metadata: creationTimestamp: "2026-06-15T06:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:56Z" name: sequence-graph-42a73-84c6b98785.18b92c6d7ee7e726 namespace: kserve-ci-e2e-test resourceVersion: "22201" uid: 52eac05b-b1be-4aaa-b026-574ecb7a92eb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-42a73 namespace: kserve-ci-e2e-test resourceVersion: "22194" uid: fa8e431f-731c-4341-bd3c-4ac8bebebb0c kind: Event lastTimestamp: "2026-06-15T06:12:56Z" message: Scaled up replica set sequence-graph-42a73-84c6b98785 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:56Z" name: sequence-graph-42a73.18b92c6d7e13a6d7 namespace: kserve-ci-e2e-test resourceVersion: "22197" uid: 63461680-f12a-4aa9-a546-9a858466842d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-42a73 namespace: kserve-ci-e2e-test resourceVersion: "22215" uid: 07ea481d-680d-4456-b19d-1aaa38695214 kind: Event lastTimestamp: "2026-06-15T06:12:56Z" message: InferenceGraph [sequence-graph-42a73] is Ready metadata: creationTimestamp: "2026-06-15T06:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:56Z" name: sequence-graph-42a73.18b92c6d830e5462 namespace: kserve-ci-e2e-test resourceVersion: "22217" uid: d8152ea4-acaf-4104-bedf-b8ad6f25e3fb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:14:06.727210Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-44470-76d55687fd-l8kcg namespace: kserve-ci-e2e-test resourceVersion: "23001" uid: fe4fe338-5d3d-4d5d-a911-91ef4d98afc2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-44470-76d55687fd-l8kcg to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:14:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:14:06Z" name: sequence-graph-44470-76d55687fd-l8kcg.18b92c7ddf18b8b3 namespace: kserve-ci-e2e-test resourceVersion: "23007" uid: 60da9bb3-a7e3-44bb-9e56-1fa293d1923a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:06Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-44470-76d55687fd-l8kcg namespace: kserve-ci-e2e-test resourceVersion: "23004" uid: fe4fe338-5d3d-4d5d-a911-91ef4d98afc2 kind: Event lastTimestamp: "2026-06-15T06:14:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-44470-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:14:06Z" name: sequence-graph-44470-76d55687fd-l8kcg.18b92c7de856d0c9 namespace: kserve-ci-e2e-test resourceVersion: "23019" uid: 8b158b20-17d3-4815-a739-1e86ca0a2c76 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:07Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-44470-76d55687fd-l8kcg namespace: kserve-ci-e2e-test resourceVersion: "23011" uid: fe4fe338-5d3d-4d5d-a911-91ef4d98afc2 kind: Event lastTimestamp: "2026-06-15T06:14:07Z" message: Add eth0 [10.133.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:14:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:14:07Z" name: sequence-graph-44470-76d55687fd-l8kcg.18b92c7e1cc44162 namespace: kserve-ci-e2e-test resourceVersion: "23028" uid: d2c4f6e3-17ba-4270-97c8-f941de8ee8e4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-44470} kind: Pod name: sequence-graph-44470-76d55687fd-l8kcg namespace: kserve-ci-e2e-test resourceVersion: "23004" uid: fe4fe338-5d3d-4d5d-a911-91ef4d98afc2 kind: Event lastTimestamp: "2026-06-15T06:14:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" already present on machine metadata: creationTimestamp: "2026-06-15T06:14:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:14:07Z" name: sequence-graph-44470-76d55687fd-l8kcg.18b92c7e1f26b7ed namespace: kserve-ci-e2e-test resourceVersion: "23030" uid: 7e1735bc-502e-4e14-9261-dece248ff530 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-44470} kind: Pod name: sequence-graph-44470-76d55687fd-l8kcg namespace: kserve-ci-e2e-test resourceVersion: "23004" uid: fe4fe338-5d3d-4d5d-a911-91ef4d98afc2 kind: Event lastTimestamp: "2026-06-15T06:14:07Z" message: 'Created container: sequence-graph-44470' metadata: creationTimestamp: "2026-06-15T06:14:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:14:07Z" name: sequence-graph-44470-76d55687fd-l8kcg.18b92c7e22836289 namespace: kserve-ci-e2e-test resourceVersion: "23032" uid: 7c5fd766-e067-49bd-9c71-285887c27ef2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-44470} kind: Pod name: sequence-graph-44470-76d55687fd-l8kcg namespace: kserve-ci-e2e-test resourceVersion: "23004" uid: fe4fe338-5d3d-4d5d-a911-91ef4d98afc2 kind: Event lastTimestamp: "2026-06-15T06:14:07Z" message: Started container sequence-graph-44470 metadata: creationTimestamp: "2026-06-15T06:14:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:14:07Z" name: sequence-graph-44470-76d55687fd-l8kcg.18b92c7e22fb4d4b namespace: kserve-ci-e2e-test resourceVersion: "23033" uid: a84e4f28-76af-4e4b-8d78-c40bec5bfbfb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-44470} kind: Pod name: sequence-graph-44470-76d55687fd-l8kcg namespace: kserve-ci-e2e-test resourceVersion: "23004" uid: fe4fe338-5d3d-4d5d-a911-91ef4d98afc2 kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: Stopping container sequence-graph-44470 metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:21Z" name: sequence-graph-44470-76d55687fd-l8kcg.18b92cf1124bcf3e namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: c5bc1f82-3e12-4d5a-b7a0-cc144400c15f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-44470} kind: Pod name: sequence-graph-44470-76d55687fd-l8kcg namespace: kserve-ci-e2e-test resourceVersion: "23004" uid: fe4fe338-5d3d-4d5d-a911-91ef4d98afc2 kind: Event lastTimestamp: "2026-06-15T06:22:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:49Z" name: sequence-graph-44470-76d55687fd-l8kcg.18b92cf1aaa26788 namespace: kserve-ci-e2e-test resourceVersion: "27453" uid: c7a4a997-7fc0-4ada-93fa-00c53cd073a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-44470-76d55687fd namespace: kserve-ci-e2e-test resourceVersion: "22998" uid: 6d5b8c67-3c6f-4d18-ab4c-bff866690f24 kind: Event lastTimestamp: "2026-06-15T06:14:06Z" message: 'Created pod: sequence-graph-44470-76d55687fd-l8kcg' metadata: creationTimestamp: "2026-06-15T06:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:06Z" name: sequence-graph-44470-76d55687fd.18b92c7dde6f588d namespace: kserve-ci-e2e-test resourceVersion: "23002" uid: 4af4a74c-5871-4a36-97a6-2eef427a9999 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-44470 namespace: kserve-ci-e2e-test resourceVersion: "22996" uid: 3de92650-95da-441a-8ab4-a418e262ba36 kind: Event lastTimestamp: "2026-06-15T06:14:06Z" message: Scaled up replica set sequence-graph-44470-76d55687fd from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:06Z" name: sequence-graph-44470.18b92c7ddd7ee11b namespace: kserve-ci-e2e-test resourceVersion: "22999" uid: 11a01853-0929-4579-8f99-0d4cbfc96754 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-44470 namespace: kserve-ci-e2e-test resourceVersion: "23015" uid: c3c9359c-6968-4db9-ad94-0e36d6b5d060 kind: Event lastTimestamp: "2026-06-15T06:14:06Z" message: InferenceGraph [sequence-graph-44470] is Ready metadata: creationTimestamp: "2026-06-15T06:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:14:06Z" name: sequence-graph-44470.18b92c7de305d39b namespace: kserve-ci-e2e-test resourceVersion: "23017" uid: 04bac13d-07a9-485f-b3b1-7c1d5e30b6dd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-44470 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 190aed00-69fd-4902-98d3-4c03fdb7bd65 kind: Event lastTimestamp: "2026-06-15T06:14:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:14:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:21Z" name: sequence-graph-44470.18b92c815c1107d1 namespace: kserve-ci-e2e-test resourceVersion: "23138" uid: 2b0c2740-8291-451d-a5e0-1e6d1c4d4459 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-44470 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 190aed00-69fd-4902-98d3-4c03fdb7bd65 kind: Event lastTimestamp: "2026-06-15T06:14:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:14:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:21Z" name: sequence-graph-44470.18b92c815c80ac1c namespace: kserve-ci-e2e-test resourceVersion: "23140" uid: d8923906-9be1-48d8-88e8-962beb07c78a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-44470 namespace: kserve-ci-e2e-test resourceVersion: "23139" uid: 190aed00-69fd-4902-98d3-4c03fdb7bd65 kind: Event lastTimestamp: "2026-06-15T06:14:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:36Z" name: sequence-graph-44470.18b92c84db3aa4f4 namespace: kserve-ci-e2e-test resourceVersion: "23245" uid: 066ef86d-37e7-47fa-a072-922c440bdd97 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:14:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-44470 namespace: kserve-ci-e2e-test resourceVersion: "23139" uid: 190aed00-69fd-4902-98d3-4c03fdb7bd65 kind: Event lastTimestamp: "2026-06-15T06:14:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:36Z" name: sequence-graph-44470.18b92c84dbadf96d namespace: kserve-ci-e2e-test resourceVersion: "23247" uid: 16b237e8-b240-4b08-874a-96d8adf2013a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:22:25.928734Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5c5b6-745649d7db-gtzw9 namespace: kserve-ci-e2e-test resourceVersion: "27095" uid: bd74db9f-d5d3-4f88-aaf8-6b6417c07424 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-5c5b6-745649d7db-gtzw9 to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:22:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:22:25Z" name: splitter-graph-5c5b6-745649d7db-gtzw9.18b92cf219d3899d namespace: kserve-ci-e2e-test resourceVersion: "27101" uid: 86e761c4-5f8c-40b5-9702-14aff6b8e922 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:26Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5c5b6-745649d7db-gtzw9 namespace: kserve-ci-e2e-test resourceVersion: "27097" uid: bd74db9f-d5d3-4f88-aaf8-6b6417c07424 kind: Event lastTimestamp: "2026-06-15T06:22:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-5c5b6-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:26Z" name: splitter-graph-5c5b6-745649d7db-gtzw9.18b92cf228244351 namespace: kserve-ci-e2e-test resourceVersion: "27114" uid: 1814f82c-5621-4f92-b8db-968a79760981 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:26Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5c5b6-745649d7db-gtzw9 namespace: kserve-ci-e2e-test resourceVersion: "27105" uid: bd74db9f-d5d3-4f88-aaf8-6b6417c07424 kind: Event lastTimestamp: "2026-06-15T06:22:26Z" message: Add eth0 [10.133.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:22:26Z" name: splitter-graph-5c5b6-745649d7db-gtzw9.18b92cf257db0739 namespace: kserve-ci-e2e-test resourceVersion: "27121" uid: 515746c6-6910-4be5-935c-e575f9e75fad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5c5b6} kind: Pod name: splitter-graph-5c5b6-745649d7db-gtzw9 namespace: kserve-ci-e2e-test resourceVersion: "27097" uid: bd74db9f-d5d3-4f88-aaf8-6b6417c07424 kind: Event lastTimestamp: "2026-06-15T06:22:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" already present on machine metadata: creationTimestamp: "2026-06-15T06:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:26Z" name: splitter-graph-5c5b6-745649d7db-gtzw9.18b92cf259332d29 namespace: kserve-ci-e2e-test resourceVersion: "27123" uid: 4fa8be02-dc44-418a-9dd2-75bbd56cdaf6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5c5b6} kind: Pod name: splitter-graph-5c5b6-745649d7db-gtzw9 namespace: kserve-ci-e2e-test resourceVersion: "27097" uid: bd74db9f-d5d3-4f88-aaf8-6b6417c07424 kind: Event lastTimestamp: "2026-06-15T06:22:27Z" message: 'Created container: splitter-graph-5c5b6' metadata: creationTimestamp: "2026-06-15T06:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:27Z" name: splitter-graph-5c5b6-745649d7db-gtzw9.18b92cf25ca70394 namespace: kserve-ci-e2e-test resourceVersion: "27124" uid: a1528bad-589f-4d0b-ac38-11453ad64d04 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5c5b6} kind: Pod name: splitter-graph-5c5b6-745649d7db-gtzw9 namespace: kserve-ci-e2e-test resourceVersion: "27097" uid: bd74db9f-d5d3-4f88-aaf8-6b6417c07424 kind: Event lastTimestamp: "2026-06-15T06:22:27Z" message: Started container splitter-graph-5c5b6 metadata: creationTimestamp: "2026-06-15T06:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:27Z" name: splitter-graph-5c5b6-745649d7db-gtzw9.18b92cf25d22569a namespace: kserve-ci-e2e-test resourceVersion: "27125" uid: 6ed2ed9c-271d-4441-b88f-2b059db8358d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5c5b6} kind: Pod name: splitter-graph-5c5b6-745649d7db-gtzw9 namespace: kserve-ci-e2e-test resourceVersion: "27097" uid: bd74db9f-d5d3-4f88-aaf8-6b6417c07424 kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: Stopping container splitter-graph-5c5b6 metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:36Z" name: splitter-graph-5c5b6-745649d7db-gtzw9.18b92cf474374b16 namespace: kserve-ci-e2e-test resourceVersion: "27217" uid: 3a9e5cf4-46f5-4572-b914-02c16a7815e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5c5b6} kind: Pod name: splitter-graph-5c5b6-745649d7db-gtzw9 namespace: kserve-ci-e2e-test resourceVersion: "27097" uid: bd74db9f-d5d3-4f88-aaf8-6b6417c07424 kind: Event lastTimestamp: "2026-06-15T06:23:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:23:03Z" name: splitter-graph-5c5b6-745649d7db-gtzw9.18b92cf4fbc34145 namespace: kserve-ci-e2e-test resourceVersion: "27573" uid: d4e1e1ad-efdf-44a6-b567-27c3e5e8cf31 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-5c5b6-745649d7db namespace: kserve-ci-e2e-test resourceVersion: "27092" uid: f54969ca-2f85-4d6e-a8c9-f11b4b4e7b06 kind: Event lastTimestamp: "2026-06-15T06:22:25Z" message: 'Created pod: splitter-graph-5c5b6-745649d7db-gtzw9' metadata: creationTimestamp: "2026-06-15T06:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:25Z" name: splitter-graph-5c5b6-745649d7db.18b92cf219416904 namespace: kserve-ci-e2e-test resourceVersion: "27096" uid: 216ab2a4-f477-4532-a921-222403a6c181 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-5c5b6 namespace: kserve-ci-e2e-test resourceVersion: "27090" uid: 5fff625a-a93c-43d7-80ee-1c3887dc5b33 kind: Event lastTimestamp: "2026-06-15T06:22:25Z" message: Scaled up replica set splitter-graph-5c5b6-745649d7db from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:25Z" name: splitter-graph-5c5b6.18b92cf218a00930 namespace: kserve-ci-e2e-test resourceVersion: "27093" uid: 2d703555-7700-469a-ac73-8a9697c056ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-5c5b6 namespace: kserve-ci-e2e-test resourceVersion: "27109" uid: df4f6e42-df87-493e-8b04-bf905fbc1c0e kind: Event lastTimestamp: "2026-06-15T06:22:25Z" message: InferenceGraph [splitter-graph-5c5b6] is Ready metadata: creationTimestamp: "2026-06-15T06:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:22:25Z" name: splitter-graph-5c5b6.18b92cf21d8fae7e namespace: kserve-ci-e2e-test resourceVersion: "27111" uid: 0299dfd5-ce28-440f-9011-31cde337cefa reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:23:36.221679Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5edeb-655669497c-ssv4w namespace: kserve-ci-e2e-test resourceVersion: "27915" uid: 3dcd8814-5e58-40db-855b-470c2872a5b7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-5edeb-655669497c-ssv4w to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:23:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:23:36Z" name: splitter-graph-5edeb-655669497c-ssv4w.18b92d02779cc4c2 namespace: kserve-ci-e2e-test resourceVersion: "27921" uid: 10f9231d-7007-4e3d-ba02-6e00a0756528 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:36Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-5edeb-655669497c-ssv4w namespace: kserve-ci-e2e-test resourceVersion: "27925" uid: 3dcd8814-5e58-40db-855b-470c2872a5b7 kind: Event lastTimestamp: "2026-06-15T06:23:36Z" message: Add eth0 [10.133.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:23:36Z" name: splitter-graph-5edeb-655669497c-ssv4w.18b92d0291684381 namespace: kserve-ci-e2e-test resourceVersion: "27939" uid: 3ec2f08e-8e04-4e3d-8da1-ed0aab23e58b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5edeb} kind: Pod name: splitter-graph-5edeb-655669497c-ssv4w namespace: kserve-ci-e2e-test resourceVersion: "27918" uid: 3dcd8814-5e58-40db-855b-470c2872a5b7 kind: Event lastTimestamp: "2026-06-15T06:23:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" already present on machine metadata: creationTimestamp: "2026-06-15T06:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:23:36Z" name: splitter-graph-5edeb-655669497c-ssv4w.18b92d0292d7e6ae namespace: kserve-ci-e2e-test resourceVersion: "27941" uid: 88de1107-3346-4c4c-b6fc-7c96a67707e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5edeb} kind: Pod name: splitter-graph-5edeb-655669497c-ssv4w namespace: kserve-ci-e2e-test resourceVersion: "27918" uid: 3dcd8814-5e58-40db-855b-470c2872a5b7 kind: Event lastTimestamp: "2026-06-15T06:23:36Z" message: 'Created container: splitter-graph-5edeb' metadata: creationTimestamp: "2026-06-15T06:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:23:36Z" name: splitter-graph-5edeb-655669497c-ssv4w.18b92d0295f35f45 namespace: kserve-ci-e2e-test resourceVersion: "27948" uid: 8d59c251-8076-4a28-b4d6-2c2209f2aa8a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5edeb} kind: Pod name: splitter-graph-5edeb-655669497c-ssv4w namespace: kserve-ci-e2e-test resourceVersion: "27918" uid: 3dcd8814-5e58-40db-855b-470c2872a5b7 kind: Event lastTimestamp: "2026-06-15T06:23:36Z" message: Started container splitter-graph-5edeb metadata: creationTimestamp: "2026-06-15T06:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:23:36Z" name: splitter-graph-5edeb-655669497c-ssv4w.18b92d029667411c namespace: kserve-ci-e2e-test resourceVersion: "27950" uid: b8570792-6563-4d43-97e8-9e936c69a1d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:31:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5edeb} kind: Pod name: splitter-graph-5edeb-655669497c-ssv4w namespace: kserve-ci-e2e-test resourceVersion: "27918" uid: 3dcd8814-5e58-40db-855b-470c2872a5b7 kind: Event lastTimestamp: "2026-06-15T06:31:50Z" message: Stopping container splitter-graph-5edeb metadata: creationTimestamp: "2026-06-15T06:31:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:31:50Z" name: splitter-graph-5edeb-655669497c-ssv4w.18b92d75a60eb969 namespace: kserve-ci-e2e-test resourceVersion: "31390" uid: dc2d47c1-8665-4caa-b385-701a61bc3c0a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:31:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-5edeb} kind: Pod name: splitter-graph-5edeb-655669497c-ssv4w namespace: kserve-ci-e2e-test resourceVersion: "27918" uid: 3dcd8814-5e58-40db-855b-470c2872a5b7 kind: Event lastTimestamp: "2026-06-15T06:32:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:32:18Z" name: splitter-graph-5edeb-655669497c-ssv4w.18b92d76480b69e3 namespace: kserve-ci-e2e-test resourceVersion: "31629" uid: 588fe63b-a5f9-4eac-b7b4-91003422fb90 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-5edeb-655669497c namespace: kserve-ci-e2e-test resourceVersion: "27910" uid: 7f11439e-1261-4747-adcc-bce7f4e0bc92 kind: Event lastTimestamp: "2026-06-15T06:23:36Z" message: 'Created pod: splitter-graph-5edeb-655669497c-ssv4w' metadata: creationTimestamp: "2026-06-15T06:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:36Z" name: splitter-graph-5edeb-655669497c.18b92d02772aec33 namespace: kserve-ci-e2e-test resourceVersion: "27917" uid: e5c278e2-b2e6-4ee9-9cf7-6da746022aa8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-5edeb namespace: kserve-ci-e2e-test resourceVersion: "27908" uid: 5882b6fa-6d45-4bfe-8b40-24be37b659d4 kind: Event lastTimestamp: "2026-06-15T06:23:36Z" message: Scaled up replica set splitter-graph-5edeb-655669497c from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:36Z" name: splitter-graph-5edeb.18b92d02767c745b namespace: kserve-ci-e2e-test resourceVersion: "27912" uid: 94b386e7-ef53-4284-b75b-3c25354f6da4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-5edeb namespace: kserve-ci-e2e-test resourceVersion: "27931" uid: 76a758e6-b7c6-400a-9130-b37075322821 kind: Event lastTimestamp: "2026-06-15T06:23:36Z" message: InferenceGraph [splitter-graph-5edeb] is Ready metadata: creationTimestamp: "2026-06-15T06:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:23:36Z" name: splitter-graph-5edeb.18b92d027b8ad61d namespace: kserve-ci-e2e-test resourceVersion: "27933" uid: 497f203e-419f-45eb-b116-fbee7462e578 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:23:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5edeb namespace: kserve-ci-e2e-test resourceVersion: "27909" uid: 10976981-bf21-4616-b4d7-993bb3194c00 kind: Event lastTimestamp: "2026-06-15T06:24:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:24:21Z" name: splitter-graph-5edeb.18b92d05f5177c52 namespace: kserve-ci-e2e-test resourceVersion: "28278" uid: 9b88c87e-c0bf-46e0-a705-1ed7a28e6758 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:23:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-5edeb namespace: kserve-ci-e2e-test resourceVersion: "27909" uid: 10976981-bf21-4616-b4d7-993bb3194c00 kind: Event lastTimestamp: "2026-06-15T06:24:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:24:21Z" name: splitter-graph-5edeb.18b92d05f57d5bc2 namespace: kserve-ci-e2e-test resourceVersion: "28279" uid: 6868b545-fe0c-43fd-9869-f74c6ea50c14 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:02:41.495494Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h namespace: kserve-ci-e2e-test resourceVersion: "17106" uid: 5993ef3c-639e-43b7-9d21-d54e0c592262 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:02:41Z" name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h.18b92bde541a80e7 namespace: kserve-ci-e2e-test resourceVersion: "17120" uid: 47fcb2ae-7899-47cc-8a42-87b58e99ecd4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h namespace: kserve-ci-e2e-test resourceVersion: "17132" uid: 5993ef3c-639e-43b7-9d21-d54e0c592262 kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:02:41Z" name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h.18b92bde6ec38433 namespace: kserve-ci-e2e-test resourceVersion: "17183" uid: 6056ee75-4842-4bbe-9bac-a8676fe85cfd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 5993ef3c-639e-43b7-9d21-d54e0c592262 kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:41Z" name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h.18b92bde70285201 namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: 123fabe6-69bb-4575-a857-3b2bc5779642 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 5993ef3c-639e-43b7-9d21-d54e0c592262 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h.18b92bde7334cca9 namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: d621f1a2-5915-4038-b036-ae7e0b978cd9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 5993ef3c-639e-43b7-9d21-d54e0c592262 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h.18b92bde73b65718 namespace: kserve-ci-e2e-test resourceVersion: "17188" uid: 1186476a-a4b6-4ed2-974d-8d4f54d3597e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 5993ef3c-639e-43b7-9d21-d54e0c592262 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h.18b92bde73c02a1e namespace: kserve-ci-e2e-test resourceVersion: "17189" uid: dc4e25d6-9da6-489a-8748-d27d6c3c335b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 5993ef3c-639e-43b7-9d21-d54e0c592262 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h.18b92bde76a58422 namespace: kserve-ci-e2e-test resourceVersion: "17190" uid: c26b5c75-3aef-4d00-af3e-29fd5399a26f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 5993ef3c-639e-43b7-9d21-d54e0c592262 kind: Event lastTimestamp: "2026-06-15T06:02:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:42Z" name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h.18b92bde771c6d4b namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: 876d7a60-35b7-4fcb-b656-74e55a24b1ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-06-15T06:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 5993ef3c-639e-43b7-9d21-d54e0c592262 kind: Event lastTimestamp: "2026-06-15T06:11:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:58Z" name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h.18b92bdeab702fb0 namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 90a6f9c1-c648-419c-b943-f31068cc05d3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 5993ef3c-639e-43b7-9d21-d54e0c592262 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h.18b92c5f842a0a6a namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: f0aa5dbe-c908-498a-bcf7-da443d1dcab8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 5993ef3c-639e-43b7-9d21-d54e0c592262 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h.18b92c5f842acaec namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 30841965-c7ed-478a-aa69-8aed2524c61e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 5993ef3c-639e-43b7-9d21-d54e0c592262 kind: Event lastTimestamp: "2026-06-15T06:11:58Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:58Z" name: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h.18b92c601fde7aa7 namespace: kserve-ci-e2e-test resourceVersion: "21507" uid: 86fb6ffc-d5f7-4453-84b5-38c3fd1eae67 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0ac5a-predictor-6867d6c9b namespace: kserve-ci-e2e-test resourceVersion: "17089" uid: 6aa6df6c-a7fa-420b-9ad1-579975853d0a kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: 'Created pod: success-200-isvc-0ac5a-predictor-6867d6c9b-49h5h' metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:41Z" name: success-200-isvc-0ac5a-predictor-6867d6c9b.18b92bde539174e7 namespace: kserve-ci-e2e-test resourceVersion: "17112" uid: 09153e03-9d33-4d2f-8b07-9eda29bfcbee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17081" uid: b1ade4f8-57eb-4a83-9be5-2769e2c555ab kind: Event lastTimestamp: "2026-06-15T06:02:41Z" message: Scaled up replica set success-200-isvc-0ac5a-predictor-6867d6c9b from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:41Z" name: success-200-isvc-0ac5a-predictor.18b92bde50fa548f namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: fab309b7-583d-4fcc-889b-c0fb56b9524a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:02:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17083" uid: 9e5d4f66-ae78-4512-b22f-b5b1764135b0 kind: Event lastTimestamp: "2026-06-15T06:03:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:02:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:11Z" name: success-200-isvc-0ac5a-predictor.18b92be1cf37eab5 namespace: kserve-ci-e2e-test resourceVersion: "17500" uid: e31e3578-2cc3-4992-bdbb-8c8c76684f11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:02:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17083" uid: 9e5d4f66-ae78-4512-b22f-b5b1764135b0 kind: Event lastTimestamp: "2026-06-15T06:03:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:02:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:11Z" name: success-200-isvc-0ac5a-predictor.18b92be1cfa1eb9b namespace: kserve-ci-e2e-test resourceVersion: "17501" uid: 1bd7a605-80c1-47f0-8447-fb0bc27cbe92 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:03:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17385" uid: 9e5d4f66-ae78-4512-b22f-b5b1764135b0 kind: Event lastTimestamp: "2026-06-15T06:03:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:03:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:56Z" name: success-200-isvc-0ac5a-predictor.18b92be8cd3c7991 namespace: kserve-ci-e2e-test resourceVersion: "17888" uid: 72db6ca8-399a-465d-99f4-1ac5ccf995e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:03:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0ac5a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17385" uid: 9e5d4f66-ae78-4512-b22f-b5b1764135b0 kind: Event lastTimestamp: "2026-06-15T06:03:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:03:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:56Z" name: success-200-isvc-0ac5a-predictor.18b92be8cda7a200 namespace: kserve-ci-e2e-test resourceVersion: "17889" uid: 56fca0b3-a20f-46b5-9781-41e51a563436 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:02:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0ac5a namespace: kserve-ci-e2e-test resourceVersion: "17067" uid: c70239fe-d3ee-49ad-ae8e-1dbecb935b4b kind: Event lastTimestamp: "2026-06-15T06:03:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:02:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:03:29Z" name: success-200-isvc-0ac5a.18b92bde507f189e namespace: kserve-ci-e2e-test resourceVersion: "17656" uid: aa23db93-7a2c-4f95-8c79-51dfa0740233 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0ac5a namespace: kserve-ci-e2e-test resourceVersion: "17654" uid: c70239fe-d3ee-49ad-ae8e-1dbecb935b4b kind: Event lastTimestamp: "2026-06-15T06:03:29Z" message: InferenceService [success-200-isvc-0ac5a] is Ready metadata: creationTimestamp: "2026-06-15T06:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:03:29Z" name: success-200-isvc-0ac5a.18b92be96756ea4e namespace: kserve-ci-e2e-test resourceVersion: "17655" uid: 132f48b7-bb2e-456e-8fb2-c0457146d391 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:11:56.369977Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-42a73-predictor-5854874678-fr4b4 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: c6910f7e-ced0-4964-aa33-b9b82d3bf482 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-42a73-predictor-5854874678-fr4b4 to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-42a73-predictor-5854874678-fr4b4.18b92c5f853349dd namespace: kserve-ci-e2e-test resourceVersion: "21408" uid: 470afe3b-39c3-4e7f-9c07-2806dc38032f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-42a73-predictor-5854874678-fr4b4 namespace: kserve-ci-e2e-test resourceVersion: "21415" uid: c6910f7e-ced0-4964-aa33-b9b82d3bf482 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-42a73-predictor-5854874678-fr4b4.18b92c5f9f61e651 namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: bf6bb24e-7e07-4f32-89f4-3b001136dacc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42a73-predictor-5854874678-fr4b4 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: c6910f7e-ced0-4964-aa33-b9b82d3bf482 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-42a73-predictor-5854874678-fr4b4.18b92c5fa0baafe5 namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: e7056be9-9b2d-48ca-aae8-712716a40c07 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42a73-predictor-5854874678-fr4b4 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: c6910f7e-ced0-4964-aa33-b9b82d3bf482 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-42a73-predictor-5854874678-fr4b4.18b92c5fa42788c7 namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: f005de72-d0a7-4307-9d95-ba4ac96664d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42a73-predictor-5854874678-fr4b4 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: c6910f7e-ced0-4964-aa33-b9b82d3bf482 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-42a73-predictor-5854874678-fr4b4.18b92c5fa4b52487 namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: 11bc541b-294b-4ad1-a43b-f2e10a796459 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42a73-predictor-5854874678-fr4b4 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: c6910f7e-ced0-4964-aa33-b9b82d3bf482 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-42a73-predictor-5854874678-fr4b4.18b92c5fa4bf48b0 namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: f05bee4b-9a5c-43c3-bfba-b91554f550d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42a73-predictor-5854874678-fr4b4 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: c6910f7e-ced0-4964-aa33-b9b82d3bf482 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-42a73-predictor-5854874678-fr4b4.18b92c5fa920e065 namespace: kserve-ci-e2e-test resourceVersion: "21471" uid: 6989cd9d-488f-413b-ac63-b3b3eb8fa04b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42a73-predictor-5854874678-fr4b4 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: c6910f7e-ced0-4964-aa33-b9b82d3bf482 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-42a73-predictor-5854874678-fr4b4.18b92c5fa99b6cb6 namespace: kserve-ci-e2e-test resourceVersion: "21472" uid: f7eacac9-8de2-4454-a0dc-1d25a75f9316 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:11:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42a73-predictor-5854874678-fr4b4 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: c6910f7e-ced0-4964-aa33-b9b82d3bf482 kind: Event lastTimestamp: "2026-06-15T06:12:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:11:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:33Z" name: success-200-isvc-42a73-predictor-5854874678-fr4b4.18b92c5fc4960285 namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: 39e7b202-ce00-453c-bc74-a6268461791b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-42a73-predictor-5854874678-fr4b4 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: c6910f7e-ced0-4964-aa33-b9b82d3bf482 kind: Event lastTimestamp: "2026-06-15T06:13:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:06Z" name: success-200-isvc-42a73-predictor-5854874678-fr4b4.18b92c6fe146df61 namespace: kserve-ci-e2e-test resourceVersion: "22392" uid: 65c03d5f-c941-4659-9934-d5002bf9dba1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42a73-predictor-5854874678-fr4b4 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: c6910f7e-ced0-4964-aa33-b9b82d3bf482 kind: Event lastTimestamp: "2026-06-15T06:13:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:06Z" name: success-200-isvc-42a73-predictor-5854874678-fr4b4.18b92c6fe1735c32 namespace: kserve-ci-e2e-test resourceVersion: "22395" uid: 5f1f040c-4eeb-4f51-9375-ef2efc73c6cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-42a73-predictor-5854874678-fr4b4 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: c6910f7e-ced0-4964-aa33-b9b82d3bf482 kind: Event lastTimestamp: "2026-06-15T06:13:08Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:13:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:08Z" name: success-200-isvc-42a73-predictor-5854874678-fr4b4.18b92c704cdfce36 namespace: kserve-ci-e2e-test resourceVersion: "22473" uid: 1b48ea59-19e3-427f-9d53-5efe8ae34524 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-42a73-predictor-5854874678 namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: 8000821b-a8c0-43ce-bb79-aec0dfff22c0 kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: 'Created pod: success-200-isvc-42a73-predictor-5854874678-fr4b4' metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-42a73-predictor-5854874678.18b92c5f84bcf6e4 namespace: kserve-ci-e2e-test resourceVersion: "21404" uid: 0ee9d3a8-a0ff-4db7-842a-5d19eeb5bfec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-42a73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: 31f542c7-f371-4e5a-9606-faaf4a46566e kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: Scaled up replica set success-200-isvc-42a73-predictor-5854874678 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-42a73-predictor.18b92c5f82a19675 namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: 626e4f11-a7b5-4120-b91e-062609457c71 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-42a73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: f9f9b67e-940f-4cfb-bfaf-33b0a37e089a kind: Event lastTimestamp: "2026-06-15T06:12:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:26Z" name: success-200-isvc-42a73-predictor.18b92c63015aff6e namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: fcf69430-9d13-4151-8ff9-197037f61156 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-42a73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: f9f9b67e-940f-4cfb-bfaf-33b0a37e089a kind: Event lastTimestamp: "2026-06-15T06:12:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:26Z" name: success-200-isvc-42a73-predictor.18b92c6301cc8139 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: 06269238-a021-4f3d-b087-9443888600f6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:12:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-42a73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21766" uid: f9f9b67e-940f-4cfb-bfaf-33b0a37e089a kind: Event lastTimestamp: "2026-06-15T06:12:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:56Z" name: success-200-isvc-42a73-predictor.18b92c69ff76479c namespace: kserve-ci-e2e-test resourceVersion: "22200" uid: d6253659-9539-428f-9cbb-6a576d18a4ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:12:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-42a73-predictor namespace: kserve-ci-e2e-test resourceVersion: "21766" uid: f9f9b67e-940f-4cfb-bfaf-33b0a37e089a kind: Event lastTimestamp: "2026-06-15T06:12:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:56Z" name: success-200-isvc-42a73-predictor.18b92c69ffe399ad namespace: kserve-ci-e2e-test resourceVersion: "22206" uid: 493fe46f-cfa5-438d-835b-4e96c8e5848e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-42a73 namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: 687d04f4-6029-4428-8888-9aabea766e3f kind: Event lastTimestamp: "2026-06-15T06:12:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:43Z" name: success-200-isvc-42a73.18b92c5f8280c204 namespace: kserve-ci-e2e-test resourceVersion: "22085" uid: 5ca3edf6-9800-4123-82f7-c7c96a05c38e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-42a73 namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: 687d04f4-6029-4428-8888-9aabea766e3f kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: 'Failed to update status for InferenceService "success-200-isvc-42a73": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-42a73": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-42a73.18b92c5f833ea182 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 2a1d96d5-7ed5-40a8-95f1-dbb4e487805f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-42a73 namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: 687d04f4-6029-4428-8888-9aabea766e3f kind: Event lastTimestamp: "2026-06-15T06:11:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-42a73": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:56Z" name: success-200-isvc-42a73.18b92c5f833ef376 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: ca5872e7-8ee9-4fa8-bbf4-91ace221f3f5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-42a73 namespace: kserve-ci-e2e-test resourceVersion: "22082" uid: 687d04f4-6029-4428-8888-9aabea766e3f kind: Event lastTimestamp: "2026-06-15T06:12:43Z" message: InferenceService [success-200-isvc-42a73] is Ready metadata: creationTimestamp: "2026-06-15T06:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:43Z" name: success-200-isvc-42a73.18b92c6a7fc0d7ef namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: e123f18a-d3ad-43c3-ad4f-5bf349c2ceaa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:13:06.633929Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-44470-predictor-5787bff68b-pcp4k namespace: kserve-ci-e2e-test resourceVersion: "22382" uid: 33cd3e46-5e4b-457a-b235-ea70b284a6b0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-44470-predictor-5787bff68b-pcp4k to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:13:06Z" name: success-200-isvc-44470-predictor-5787bff68b-pcp4k.18b92c6fe1421b70 namespace: kserve-ci-e2e-test resourceVersion: "22390" uid: 63770c5f-1ff8-4ef3-a3b8-e5377ba36c6f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-44470-predictor-5787bff68b-pcp4k namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 33cd3e46-5e4b-457a-b235-ea70b284a6b0 kind: Event lastTimestamp: "2026-06-15T06:13:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-44470-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:06Z" name: success-200-isvc-44470-predictor-5787bff68b-pcp4k.18b92c6fef6dfd2f namespace: kserve-ci-e2e-test resourceVersion: "22427" uid: 0cec4550-c9bb-48b8-bd03-920f910d7397 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-44470-predictor-5787bff68b-pcp4k namespace: kserve-ci-e2e-test resourceVersion: "22398" uid: 33cd3e46-5e4b-457a-b235-ea70b284a6b0 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:13:07Z" name: success-200-isvc-44470-predictor-5787bff68b-pcp4k.18b92c70201d0edd namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: ba446b70-caa6-45f5-ae35-9a43b6c94e92 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44470-predictor-5787bff68b-pcp4k namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 33cd3e46-5e4b-457a-b235-ea70b284a6b0 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:07Z" name: success-200-isvc-44470-predictor-5787bff68b-pcp4k.18b92c7021ab7b9e namespace: kserve-ci-e2e-test resourceVersion: "22445" uid: b29ccdf1-8eee-49f2-87ed-61479ec5e820 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44470-predictor-5787bff68b-pcp4k namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 33cd3e46-5e4b-457a-b235-ea70b284a6b0 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:07Z" name: success-200-isvc-44470-predictor-5787bff68b-pcp4k.18b92c7024e8157c namespace: kserve-ci-e2e-test resourceVersion: "22447" uid: 5886f650-9d1d-409e-b8c0-3f796f927d0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44470-predictor-5787bff68b-pcp4k namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 33cd3e46-5e4b-457a-b235-ea70b284a6b0 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:07Z" name: success-200-isvc-44470-predictor-5787bff68b-pcp4k.18b92c702565c5e8 namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: ec3ac255-cda6-473c-ae8c-7edfd6444507 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44470-predictor-5787bff68b-pcp4k namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 33cd3e46-5e4b-457a-b235-ea70b284a6b0 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:07Z" name: success-200-isvc-44470-predictor-5787bff68b-pcp4k.18b92c70256fb2a8 namespace: kserve-ci-e2e-test resourceVersion: "22450" uid: 718371a7-0566-4191-ad97-97643404dddc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44470-predictor-5787bff68b-pcp4k namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 33cd3e46-5e4b-457a-b235-ea70b284a6b0 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:07Z" name: success-200-isvc-44470-predictor-5787bff68b-pcp4k.18b92c702859cf06 namespace: kserve-ci-e2e-test resourceVersion: "22453" uid: b7b47f85-a64f-4f90-9b51-22284d7b5cb2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44470-predictor-5787bff68b-pcp4k namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 33cd3e46-5e4b-457a-b235-ea70b284a6b0 kind: Event lastTimestamp: "2026-06-15T06:13:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:13:07Z" name: success-200-isvc-44470-predictor-5787bff68b-pcp4k.18b92c7028dcbd88 namespace: kserve-ci-e2e-test resourceVersion: "22454" uid: 4849803c-978e-4862-8b3a-5938cb8310f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-06-15T06:13:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44470-predictor-5787bff68b-pcp4k namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 33cd3e46-5e4b-457a-b235-ea70b284a6b0 kind: Event lastTimestamp: "2026-06-15T06:22:24Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:13:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:24Z" name: success-200-isvc-44470-predictor-5787bff68b-pcp4k.18b92c7061a82b8e namespace: kserve-ci-e2e-test resourceVersion: "27065" uid: 42a36144-246f-416b-9e8a-85fa89e1db88 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44470-predictor-5787bff68b-pcp4k namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 33cd3e46-5e4b-457a-b235-ea70b284a6b0 kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:21Z" name: success-200-isvc-44470-predictor-5787bff68b-pcp4k.18b92cf119653c77 namespace: kserve-ci-e2e-test resourceVersion: "26951" uid: 7d223ab4-6584-437e-9855-3e6857552e54 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44470-predictor-5787bff68b-pcp4k namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 33cd3e46-5e4b-457a-b235-ea70b284a6b0 kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:21Z" name: success-200-isvc-44470-predictor-5787bff68b-pcp4k.18b92cf11965f14e namespace: kserve-ci-e2e-test resourceVersion: "26953" uid: 1e6f0837-26fa-4cc5-9964-20b3d50cc96e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44470-predictor-5787bff68b-pcp4k namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 33cd3e46-5e4b-457a-b235-ea70b284a6b0 kind: Event lastTimestamp: "2026-06-15T06:22:24Z" message: 'Readiness probe failed: Get "https://10.133.0.44:8643/healthz": dial tcp 10.133.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:24Z" name: success-200-isvc-44470-predictor-5787bff68b-pcp4k.18b92cf1d6125d18 namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: 1fcdef4d-4abe-4f31-8caf-fd65f32b1c30 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-44470-predictor-5787bff68b namespace: kserve-ci-e2e-test resourceVersion: "22365" uid: 2c29b926-d980-4ee5-9297-318f768f4cdd kind: Event lastTimestamp: "2026-06-15T06:13:06Z" message: 'Created pod: success-200-isvc-44470-predictor-5787bff68b-pcp4k' metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:06Z" name: success-200-isvc-44470-predictor-5787bff68b.18b92c6fe0aac6ca namespace: kserve-ci-e2e-test resourceVersion: "22386" uid: a2763bda-82a6-49db-aa1c-167ad8a3f165 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-44470-predictor namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: b6800f07-3412-4f2a-8c30-586b9c272624 kind: Event lastTimestamp: "2026-06-15T06:13:06Z" message: Scaled up replica set success-200-isvc-44470-predictor-5787bff68b from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:06Z" name: success-200-isvc-44470-predictor.18b92c6fdee4b7f6 namespace: kserve-ci-e2e-test resourceVersion: "22368" uid: b02ae86d-ec93-41a3-9b3f-e5cede1fde7b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44470-predictor namespace: kserve-ci-e2e-test resourceVersion: "22364" uid: ebc1d0ef-3a59-47cb-9900-f563aac9a300 kind: Event lastTimestamp: "2026-06-15T06:13:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:21Z" name: success-200-isvc-44470-predictor.18b92c735d990ac6 namespace: kserve-ci-e2e-test resourceVersion: "22629" uid: 7ef5afc2-1c80-4ca5-98d1-6d6d8c1b8868 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44470-predictor namespace: kserve-ci-e2e-test resourceVersion: "22364" uid: ebc1d0ef-3a59-47cb-9900-f563aac9a300 kind: Event lastTimestamp: "2026-06-15T06:13:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:21Z" name: success-200-isvc-44470-predictor.18b92c735e12830b namespace: kserve-ci-e2e-test resourceVersion: "22631" uid: 182a4de6-a56a-467d-bcb2-f4e9592410ef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:13:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44470-predictor namespace: kserve-ci-e2e-test resourceVersion: "22630" uid: ebc1d0ef-3a59-47cb-9900-f563aac9a300 kind: Event lastTimestamp: "2026-06-15T06:14:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:36Z" name: success-200-isvc-44470-predictor.18b92c76dd751943 namespace: kserve-ci-e2e-test resourceVersion: "23243" uid: 6363d2f9-24f0-4981-8bd6-631f02ca86eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:13:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44470-predictor namespace: kserve-ci-e2e-test resourceVersion: "22630" uid: ebc1d0ef-3a59-47cb-9900-f563aac9a300 kind: Event lastTimestamp: "2026-06-15T06:14:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:14:36Z" name: success-200-isvc-44470-predictor.18b92c76ddeeac7b namespace: kserve-ci-e2e-test resourceVersion: "23244" uid: d3506f7b-2a01-42cf-bcdb-00062b385696 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:13:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-44470 namespace: kserve-ci-e2e-test resourceVersion: "22342" uid: 1c9c19ee-f69f-47b0-978e-51d709807d62 kind: Event lastTimestamp: "2026-06-15T06:13:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:13:54Z" name: success-200-isvc-44470.18b92c6fdede4583 namespace: kserve-ci-e2e-test resourceVersion: "22903" uid: 12ce13d7-a3ef-48d9-8d77-4f90a2ea6c54 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:13:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-44470 namespace: kserve-ci-e2e-test resourceVersion: "22901" uid: 1c9c19ee-f69f-47b0-978e-51d709807d62 kind: Event lastTimestamp: "2026-06-15T06:13:54Z" message: InferenceService [success-200-isvc-44470] is Ready metadata: creationTimestamp: "2026-06-15T06:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:13:54Z" name: success-200-isvc-44470.18b92c7b1c96cefc namespace: kserve-ci-e2e-test resourceVersion: "22902" uid: d112eb9c-f68f-4c10-b275-964765421ae6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:21:25.891399Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: b87b98ed-848e-44ec-b2a8-f2b5bf5a222d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:21:25Z" name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h.18b92ce41f527fd9 namespace: kserve-ci-e2e-test resourceVersion: "26405" uid: c33e68f9-6517-4e56-97f1-e380485970e9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: b87b98ed-848e-44ec-b2a8-f2b5bf5a222d kind: Event lastTimestamp: "2026-06-15T06:21:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-5c5b6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:26Z" name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h.18b92ce42a97f6e2 namespace: kserve-ci-e2e-test resourceVersion: "26431" uid: 96a5f84b-f679-4eed-92b0-d076742b608b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h namespace: kserve-ci-e2e-test resourceVersion: "26413" uid: b87b98ed-848e-44ec-b2a8-f2b5bf5a222d kind: Event lastTimestamp: "2026-06-15T06:21:26Z" message: Add eth0 [10.133.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:21:26Z" name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h.18b92ce45d895245 namespace: kserve-ci-e2e-test resourceVersion: "26446" uid: 1530fab5-c9cf-4513-bcd9-99b5db0d0b90 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: b87b98ed-848e-44ec-b2a8-f2b5bf5a222d kind: Event lastTimestamp: "2026-06-15T06:21:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:26Z" name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h.18b92ce45ef0186d namespace: kserve-ci-e2e-test resourceVersion: "26449" uid: b9759b84-ce0c-418e-bce5-3470eb4b3e57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: b87b98ed-848e-44ec-b2a8-f2b5bf5a222d kind: Event lastTimestamp: "2026-06-15T06:21:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:27Z" name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h.18b92ce46209fbb9 namespace: kserve-ci-e2e-test resourceVersion: "26454" uid: bc354e58-08a7-4883-9005-c433ea1ef3f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: b87b98ed-848e-44ec-b2a8-f2b5bf5a222d kind: Event lastTimestamp: "2026-06-15T06:21:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:27Z" name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h.18b92ce462851c29 namespace: kserve-ci-e2e-test resourceVersion: "26455" uid: c7d605bb-767c-437b-847e-4ac1caf8b519 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: b87b98ed-848e-44ec-b2a8-f2b5bf5a222d kind: Event lastTimestamp: "2026-06-15T06:21:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:27Z" name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h.18b92ce4628ed61d namespace: kserve-ci-e2e-test resourceVersion: "26456" uid: 79d717d4-1293-4a19-a8a0-21c0f96446a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: b87b98ed-848e-44ec-b2a8-f2b5bf5a222d kind: Event lastTimestamp: "2026-06-15T06:21:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:27Z" name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h.18b92ce4657c84bb namespace: kserve-ci-e2e-test resourceVersion: "26460" uid: a0ed79ba-eeb7-456b-95b6-9ce3a7190932 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: b87b98ed-848e-44ec-b2a8-f2b5bf5a222d kind: Event lastTimestamp: "2026-06-15T06:21:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:27Z" name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h.18b92ce46612e6c9 namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 56e69368-8bbb-4137-be32-f6f2c1381f9e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:21:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: b87b98ed-848e-44ec-b2a8-f2b5bf5a222d kind: Event lastTimestamp: "2026-06-15T06:22:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:05Z" name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h.18b92ce4da13864e namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: c2ab9922-feae-4aed-845a-2652a0cc2e3b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: b87b98ed-848e-44ec-b2a8-f2b5bf5a222d kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:36Z" name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h.18b92cf479b7c9cc namespace: kserve-ci-e2e-test resourceVersion: "27248" uid: 5080634f-76fb-405a-906b-771149341bbb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: b87b98ed-848e-44ec-b2a8-f2b5bf5a222d kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:36Z" name: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h.18b92cf479b85e9f namespace: kserve-ci-e2e-test resourceVersion: "27251" uid: 5d67fca6-f8f2-44bc-8361-96adf7dd9913 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5c5b6-predictor-77d84cd86 namespace: kserve-ci-e2e-test resourceVersion: "26369" uid: 28a2bb2f-0851-4120-9f08-5a704c9f89b9 kind: Event lastTimestamp: "2026-06-15T06:21:25Z" message: 'Created pod: success-200-isvc-5c5b6-predictor-77d84cd86-hgg5h' metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:21:25Z" name: success-200-isvc-5c5b6-predictor-77d84cd86.18b92ce41ec3f586 namespace: kserve-ci-e2e-test resourceVersion: "26397" uid: ff088cc6-431b-41a4-8bac-5580f2f409da reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5c5b6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26364" uid: 82034657-6338-4fb1-b37e-abed14554506 kind: Event lastTimestamp: "2026-06-15T06:21:25Z" message: Scaled up replica set success-200-isvc-5c5b6-predictor-77d84cd86 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:21:25Z" name: success-200-isvc-5c5b6-predictor.18b92ce41b27660c namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: 0da590a7-b44a-4ea7-849c-d04e97d27750 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:21:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5c5b6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26367" uid: 389c0a44-9658-4328-b1a3-1d8d907fa475 kind: Event lastTimestamp: "2026-06-15T06:21:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:21:55Z" name: success-200-isvc-5c5b6-predictor.18b92ce799c48527 namespace: kserve-ci-e2e-test resourceVersion: "26696" uid: b9da99d8-71b6-4306-843c-93f191903919 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:21:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5c5b6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26367" uid: 389c0a44-9658-4328-b1a3-1d8d907fa475 kind: Event lastTimestamp: "2026-06-15T06:21:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:21:55Z" name: success-200-isvc-5c5b6-predictor.18b92ce79a373dfc namespace: kserve-ci-e2e-test resourceVersion: "26697" uid: 3307d2c5-2236-411e-a504-ce98018805d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:22:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5c5b6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26586" uid: 389c0a44-9658-4328-b1a3-1d8d907fa475 kind: Event lastTimestamp: "2026-06-15T06:22:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:25Z" name: success-200-isvc-5c5b6-predictor.18b92cee97bd86dc namespace: kserve-ci-e2e-test resourceVersion: "27085" uid: 2da64079-71a2-4ae6-892e-21652ed4d4bd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:22:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5c5b6-predictor namespace: kserve-ci-e2e-test resourceVersion: "26586" uid: 389c0a44-9658-4328-b1a3-1d8d907fa475 kind: Event lastTimestamp: "2026-06-15T06:22:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:25Z" name: success-200-isvc-5c5b6-predictor.18b92cee982e873b namespace: kserve-ci-e2e-test resourceVersion: "27087" uid: fd5a30fa-34f9-4cd6-856d-2d54caa15847 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:21:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5c5b6 namespace: kserve-ci-e2e-test resourceVersion: "26346" uid: c184da56-f461-4e62-abcc-9daddb8d1281 kind: Event lastTimestamp: "2026-06-15T06:22:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:22:15Z" name: success-200-isvc-5c5b6.18b92ce41af38c87 namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 4174dc57-dd44-4920-b7d2-0cd59f22ea2d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5c5b6 namespace: kserve-ci-e2e-test resourceVersion: "26854" uid: c184da56-f461-4e62-abcc-9daddb8d1281 kind: Event lastTimestamp: "2026-06-15T06:22:15Z" message: InferenceService [success-200-isvc-5c5b6] is Ready metadata: creationTimestamp: "2026-06-15T06:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:22:15Z" name: success-200-isvc-5c5b6.18b92cef9566ea1f namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: cfaf5400-0865-48f7-a614-71ee48656e83 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:00:11.452118Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15368" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:00:11Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bbb64d24894 namespace: kserve-ci-e2e-test resourceVersion: "15375" uid: e77da0de-c075-4c87-845e-c402ff1187b8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: "2026-06-15T06:00:11Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:00:11Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bbb7fd0efb8 namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: 213e5623-36ed-47a8-af57-8b7950e586ed reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: "2026-06-15T06:00:11Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1619" metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:11Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bbb8138e644 namespace: kserve-ci-e2e-test resourceVersion: "15428" uid: 36bd8c4a-10c5-4c87-8b0e-ab93bd1fe7de reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: "2026-06-15T06:00:23Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1619" in 11.395s (11.395s including waiting). Image size: 1335559793 bytes.' metadata: creationTimestamp: "2026-06-15T06:00:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:23Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bbe287092f4 namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: e980b24c-4f5d-4158-a96f-75085e5ad02e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: "2026-06-15T06:00:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:00:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:23Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bbe2c04218b namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: d8f998fa-8894-4ba0-aa63-24f719ea4d11 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: "2026-06-15T06:00:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:00:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:23Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bbe2c7feb9a namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: e20e5de4-7e5e-4a10-a693-b88b87147ada reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: "2026-06-15T06:00:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-06-15T06:00:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:23Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bbe2c8df487 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 46ba130e-2999-4206-bc9e-657d96f69e85 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: "2026-06-15T06:00:25Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.243s (2.243s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-06-15T06:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:25Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bbeb24089fe namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: 0be285f7-e1c4-48b5-90e4-9e652da969e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: "2026-06-15T06:00:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:25Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bbeb690be01 namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 7c02f5d2-ed6a-4b1b-b05e-45a00de1bb91 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: "2026-06-15T06:00:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:00:25Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bbeb70b1e64 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: cae9eadb-341f-4e1d-905a-8a5011627866 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:00:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: "2026-06-15T06:01:02Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:02Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bbedd550a6f namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: cceefd0c-b030-4704-a6bb-50b21fbbac7d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: "2026-06-15T06:01:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:45Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bd14e8ef493 namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: d59f695c-0a32-4be1-808e-5efed96374af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: "2026-06-15T06:01:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:45Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bd14e8f5db2 namespace: kserve-ci-e2e-test resourceVersion: "16440" uid: 7b5b2f7e-b4d1-4d09-8fb9-e5f3ca5f4819 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4 namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: dbd3d266-bbdf-4072-9450-37bc5ff2c4bd kind: Event lastTimestamp: "2026-06-15T06:01:47Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:47Z" name: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4.18b92bd1b9938d13 namespace: kserve-ci-e2e-test resourceVersion: "16540" uid: 15f12bd0-5b72-43ec-92f5-3c5f59ee685f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5cf27-predictor-99cc5fc54 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: 64e46ace-fdf8-4e08-a19c-90d8a6814dbe kind: Event lastTimestamp: "2026-06-15T06:00:11Z" message: 'Created pod: success-200-isvc-5cf27-predictor-99cc5fc54-gxmf4' metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:11Z" name: success-200-isvc-5cf27-predictor-99cc5fc54.18b92bbb64520e54 namespace: kserve-ci-e2e-test resourceVersion: "15370" uid: 18e78d36-debf-4a6f-8870-4aa4769b305b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5cf27-predictor namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: a558bb55-83c6-4623-864e-d42764778273 kind: Event lastTimestamp: "2026-06-15T06:00:11Z" message: Scaled up replica set success-200-isvc-5cf27-predictor-99cc5fc54 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:11Z" name: success-200-isvc-5cf27-predictor.18b92bbb613b2176 namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 52b7ad30-1a99-4cf6-80d2-3fcfe66add7a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5cf27-predictor namespace: kserve-ci-e2e-test resourceVersion: "15358" uid: cd036af3-6715-41fa-9da3-887d9041e247 kind: Event lastTimestamp: "2026-06-15T06:00:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:56Z" name: success-200-isvc-5cf27-predictor.18b92bbedfba221a namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: b7ae7d94-aa96-49dd-8a49-e800f9e83714 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:00:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5cf27-predictor namespace: kserve-ci-e2e-test resourceVersion: "15358" uid: cd036af3-6715-41fa-9da3-887d9041e247 kind: Event lastTimestamp: "2026-06-15T06:00:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:00:56Z" name: success-200-isvc-5cf27-predictor.18b92bbee023f574 namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: dec05737-f7f3-4334-b4ce-9d5079a799c2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:00:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5cf27-predictor namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: cd036af3-6715-41fa-9da3-887d9041e247 kind: Event lastTimestamp: "2026-06-15T06:01:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:41Z" name: success-200-isvc-5cf27-predictor.18b92bc25eeb5570 namespace: kserve-ci-e2e-test resourceVersion: "16340" uid: dddf3a05-bc3a-4722-8d1b-92a79fb38da2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:00:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5cf27-predictor namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: cd036af3-6715-41fa-9da3-887d9041e247 kind: Event lastTimestamp: "2026-06-15T06:01:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:41Z" name: success-200-isvc-5cf27-predictor.18b92bc25f565ee4 namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 89d37e4b-201c-4a6a-a2f7-ba7f550df59e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:00:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5cf27 namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: dc0bb8a9-6e15-4fc4-a86a-a5255476f753 kind: Event lastTimestamp: "2026-06-15T06:01:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:12Z" name: success-200-isvc-5cf27.18b92bbb611644be namespace: kserve-ci-e2e-test resourceVersion: "16076" uid: c7567304-cde7-4fbc-ab76-0b483448ff6f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5cf27 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: dc0bb8a9-6e15-4fc4-a86a-a5255476f753 kind: Event lastTimestamp: "2026-06-15T06:01:12Z" message: InferenceService [success-200-isvc-5cf27] is Ready metadata: creationTimestamp: "2026-06-15T06:01:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:12Z" name: success-200-isvc-5cf27.18b92bc99843be2d namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 2102f675-c067-4ac1-bf6d-640dca94ad2b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:22:36.150181Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9 namespace: kserve-ci-e2e-test resourceVersion: "27252" uid: d0fec994-cf65-43fc-b0a1-e5db0b83cc94 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5edeb-predictor-869ffb65f4-m57w9 to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:22:36Z" name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9.18b92cf47b127458 namespace: kserve-ci-e2e-test resourceVersion: "27258" uid: 31dc70cf-40ab-465c-b1b3-83d7ede52647 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: d0fec994-cf65-43fc-b0a1-e5db0b83cc94 kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-5edeb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:36Z" name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9.18b92cf4875f7972 namespace: kserve-ci-e2e-test resourceVersion: "27306" uid: 4613e44f-77e9-4a53-9233-0c9fe8973de6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9 namespace: kserve-ci-e2e-test resourceVersion: "27265" uid: d0fec994-cf65-43fc-b0a1-e5db0b83cc94 kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: Add eth0 [10.133.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:22:37Z" name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9.18b92cf4b9073876 namespace: kserve-ci-e2e-test resourceVersion: "27326" uid: 51d2fb99-456e-4b8d-a968-9faf43abca6e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: d0fec994-cf65-43fc-b0a1-e5db0b83cc94 kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:37Z" name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9.18b92cf4ba6a8c97 namespace: kserve-ci-e2e-test resourceVersion: "27329" uid: 9182164c-77ec-497d-93e5-4b0a90f68855 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: d0fec994-cf65-43fc-b0a1-e5db0b83cc94 kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:37Z" name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9.18b92cf4bd25f55a namespace: kserve-ci-e2e-test resourceVersion: "27330" uid: ad655859-bb50-40bb-add2-80b786731439 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: d0fec994-cf65-43fc-b0a1-e5db0b83cc94 kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:37Z" name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9.18b92cf4be27967c namespace: kserve-ci-e2e-test resourceVersion: "27332" uid: ff2f5a12-d3d1-4e7f-b088-3dac74940ca8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: d0fec994-cf65-43fc-b0a1-e5db0b83cc94 kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:37Z" name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9.18b92cf4be344a78 namespace: kserve-ci-e2e-test resourceVersion: "27334" uid: d437d05f-0275-4bda-8258-1cf780a64203 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: d0fec994-cf65-43fc-b0a1-e5db0b83cc94 kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:37Z" name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9.18b92cf4c1799de0 namespace: kserve-ci-e2e-test resourceVersion: "27337" uid: e8b60225-9372-49b6-8f49-8b3e52f2b077 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: d0fec994-cf65-43fc-b0a1-e5db0b83cc94 kind: Event lastTimestamp: "2026-06-15T06:22:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:37Z" name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9.18b92cf4c1f56903 namespace: kserve-ci-e2e-test resourceVersion: "27338" uid: 4667bdd3-214c-4491-9f2a-e674925388d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: d0fec994-cf65-43fc-b0a1-e5db0b83cc94 kind: Event lastTimestamp: "2026-06-15T06:23:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.50:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:23:14Z" name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9.18b92cf4fe762a55 namespace: kserve-ci-e2e-test resourceVersion: "27686" uid: c961cd7d-f776-4b90-8275-1b8549a86f7e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:31:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: d0fec994-cf65-43fc-b0a1-e5db0b83cc94 kind: Event lastTimestamp: "2026-06-15T06:31:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:31:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:31:51Z" name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9.18b92d75ac4e41fe namespace: kserve-ci-e2e-test resourceVersion: "31409" uid: 86b4aa62-1da8-49ae-9ae0-90a643e8636d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:31:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9 namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: d0fec994-cf65-43fc-b0a1-e5db0b83cc94 kind: Event lastTimestamp: "2026-06-15T06:31:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:31:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:31:51Z" name: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9.18b92d75ac4f10e8 namespace: kserve-ci-e2e-test resourceVersion: "31411" uid: 167c8c99-8672-4a5d-9ec8-4f81ed8653ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5edeb-predictor-869ffb65f4 namespace: kserve-ci-e2e-test resourceVersion: "27241" uid: e22f25b3-1c65-4016-b8f4-6336b3208370 kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: 'Created pod: success-200-isvc-5edeb-predictor-869ffb65f4-m57w9' metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:36Z" name: success-200-isvc-5edeb-predictor-869ffb65f4.18b92cf47a91007a namespace: kserve-ci-e2e-test resourceVersion: "27255" uid: b32f6a2a-f174-4ae8-999a-0888224d53d0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5edeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27234" uid: ba201632-5890-4e0c-9aaf-2a22a98b159f kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: Scaled up replica set success-200-isvc-5edeb-predictor-869ffb65f4 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:36Z" name: success-200-isvc-5edeb-predictor.18b92cf478f746ce namespace: kserve-ci-e2e-test resourceVersion: "27243" uid: ce972a54-46f6-42d8-9da0-44f3dea806b6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:22:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5edeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27239" uid: c6aaad68-c6de-4dac-b4ee-a66dd09c59d3 kind: Event lastTimestamp: "2026-06-15T06:23:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:21Z" name: success-200-isvc-5edeb-predictor.18b92cf7f7b10981 namespace: kserve-ci-e2e-test resourceVersion: "27737" uid: f2ddba90-3180-4836-836a-117992b0bcc3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:22:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5edeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27239" uid: c6aaad68-c6de-4dac-b4ee-a66dd09c59d3 kind: Event lastTimestamp: "2026-06-15T06:23:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:21Z" name: success-200-isvc-5edeb-predictor.18b92cf7f818b400 namespace: kserve-ci-e2e-test resourceVersion: "27738" uid: e8f086e6-76fb-4ddb-9389-942af73cb1d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:23:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5edeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27471" uid: c6aaad68-c6de-4dac-b4ee-a66dd09c59d3 kind: Event lastTimestamp: "2026-06-15T06:24:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:24:06Z" name: success-200-isvc-5edeb-predictor.18b92cfb774f7eaa namespace: kserve-ci-e2e-test resourceVersion: "28171" uid: 56ae407b-6519-4234-a3d9-cafa760d9677 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:23:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5edeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "27471" uid: c6aaad68-c6de-4dac-b4ee-a66dd09c59d3 kind: Event lastTimestamp: "2026-06-15T06:24:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:24:06Z" name: success-200-isvc-5edeb-predictor.18b92cfb77bbe85f namespace: kserve-ci-e2e-test resourceVersion: "28173" uid: bd8b692e-506f-4d80-9729-ac2dfd4beef6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5edeb namespace: kserve-ci-e2e-test resourceVersion: "27221" uid: dd4b49e4-a458-407a-84ea-279db0504b00 kind: Event lastTimestamp: "2026-06-15T06:23:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:23:24Z" name: success-200-isvc-5edeb.18b92cf479002fc9 namespace: kserve-ci-e2e-test resourceVersion: "27811" uid: 7fbe5e2b-1457-4f11-8b0e-2d638f91cc7f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5edeb namespace: kserve-ci-e2e-test resourceVersion: "27221" uid: dd4b49e4-a458-407a-84ea-279db0504b00 kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: 'Failed to update status for InferenceService "success-200-isvc-5edeb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5edeb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:22:36Z" name: success-200-isvc-5edeb.18b92cf479619ccc namespace: kserve-ci-e2e-test resourceVersion: "27247" uid: 6af33836-852a-4887-9ee2-9c8cce232c41 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5edeb namespace: kserve-ci-e2e-test resourceVersion: "27221" uid: dd4b49e4-a458-407a-84ea-279db0504b00 kind: Event lastTimestamp: "2026-06-15T06:22:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5edeb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:22:36Z" name: success-200-isvc-5edeb.18b92cf47961ed94 namespace: kserve-ci-e2e-test resourceVersion: "27249" uid: e80c3c91-fb1b-4c45-b100-c882636caa44 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5edeb namespace: kserve-ci-e2e-test resourceVersion: "27809" uid: dd4b49e4-a458-407a-84ea-279db0504b00 kind: Event lastTimestamp: "2026-06-15T06:23:24Z" message: InferenceService [success-200-isvc-5edeb] is Ready metadata: creationTimestamp: "2026-06-15T06:23:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:23:24Z" name: success-200-isvc-5edeb.18b92cffb96036c0 namespace: kserve-ci-e2e-test resourceVersion: "27810" uid: d852de84-39b5-4963-b9ee-8b70ad0b3d9a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:22:21.659288Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh namespace: kserve-ci-e2e-test resourceVersion: "26957" uid: 71fd1f5e-b994-4010-90dd-b1158035917a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6c800-predictor-866b9bb7cd-p55wh to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:22:21Z" name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh.18b92cf11b58ec8a namespace: kserve-ci-e2e-test resourceVersion: "26961" uid: 027b7196-dce7-4c81-bd02-ae06b4fc6a8d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 71fd1f5e-b994-4010-90dd-b1158035917a kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6c800-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:21Z" name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh.18b92cf1256a21a6 namespace: kserve-ci-e2e-test resourceVersion: "27008" uid: d43acf7a-b234-4779-bcb5-db77a68afaf4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh namespace: kserve-ci-e2e-test resourceVersion: "26969" uid: 71fd1f5e-b994-4010-90dd-b1158035917a kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: Add eth0 [10.133.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:22:22Z" name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh.18b92cf15916080d namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: ace5f8cf-0046-4e1e-a1f4-f414b956b9a9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 71fd1f5e-b994-4010-90dd-b1158035917a kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:22Z" name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh.18b92cf15b43b359 namespace: kserve-ci-e2e-test resourceVersion: "27025" uid: 84e612c5-13b3-4154-bd09-350bb41a61b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 71fd1f5e-b994-4010-90dd-b1158035917a kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:22Z" name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh.18b92cf15e5c02e0 namespace: kserve-ci-e2e-test resourceVersion: "27029" uid: fe315f46-9be2-4b25-bb65-a810b7b37944 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 71fd1f5e-b994-4010-90dd-b1158035917a kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:22Z" name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh.18b92cf15eded3ad namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 9be8214a-381e-4fbb-87d7-12da5a6f56fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 71fd1f5e-b994-4010-90dd-b1158035917a kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:22Z" name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh.18b92cf15eeace86 namespace: kserve-ci-e2e-test resourceVersion: "27032" uid: 190b3a2f-87e9-49a5-9587-fa00fb701d7c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 71fd1f5e-b994-4010-90dd-b1158035917a kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:22Z" name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh.18b92cf161e128d6 namespace: kserve-ci-e2e-test resourceVersion: "27034" uid: 0084e946-8e22-4eb0-a1bc-3e8abd97a3cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 71fd1f5e-b994-4010-90dd-b1158035917a kind: Event lastTimestamp: "2026-06-15T06:22:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:22:22Z" name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh.18b92cf1625f0d86 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: 8dcd32d1-f9c1-4443-be6b-ffd2005cca5c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 71fd1f5e-b994-4010-90dd-b1158035917a kind: Event lastTimestamp: "2026-06-15T06:23:00Z" message: 'Readiness probe failed: dial tcp 10.133.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:23:00Z" name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh.18b92cf1b7cb7646 namespace: kserve-ci-e2e-test resourceVersion: "27553" uid: 84fc262d-71f2-43ef-9134-37579a37f4a7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:39:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 71fd1f5e-b994-4010-90dd-b1158035917a kind: Event lastTimestamp: "2026-06-15T06:39:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:39:41Z" name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh.18b92de32a048251 namespace: kserve-ci-e2e-test resourceVersion: "34658" uid: 245b527a-c3a4-411b-9923-bb8beb6faabc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:39:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 71fd1f5e-b994-4010-90dd-b1158035917a kind: Event lastTimestamp: "2026-06-15T06:39:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:39:41Z" name: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh.18b92de32a05c9dc namespace: kserve-ci-e2e-test resourceVersion: "34659" uid: 359805fc-f28b-41cb-9160-652863bbe3fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6c800-predictor-866b9bb7cd namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 1f49159a-c4d7-42a7-a587-cf554fd8ba47 kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: 'Created pod: success-200-isvc-6c800-predictor-866b9bb7cd-p55wh' metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:21Z" name: success-200-isvc-6c800-predictor-866b9bb7cd.18b92cf11ae23a71 namespace: kserve-ci-e2e-test resourceVersion: "26958" uid: 02bcc8d9-52b3-4a7a-8c5f-9e6460add06d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6c800-predictor namespace: kserve-ci-e2e-test resourceVersion: "26938" uid: 36f41096-eec6-4455-a5cb-8887255f33ef kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: Scaled up replica set success-200-isvc-6c800-predictor-866b9bb7cd from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:22:21Z" name: success-200-isvc-6c800-predictor.18b92cf119050627 namespace: kserve-ci-e2e-test resourceVersion: "26948" uid: 254d7587-5fe9-4fc5-b94f-f2ce3981a58a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c800-predictor namespace: kserve-ci-e2e-test resourceVersion: "26943" uid: fb25ba57-483f-4113-9a39-2eaaa229b23b kind: Event lastTimestamp: "2026-06-15T06:23:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:06Z" name: success-200-isvc-6c800-predictor.18b92cf4977f0ace namespace: kserve-ci-e2e-test resourceVersion: "27608" uid: 0022f635-4641-4528-af25-4d15cb1f5354 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:22:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c800-predictor namespace: kserve-ci-e2e-test resourceVersion: "26943" uid: fb25ba57-483f-4113-9a39-2eaaa229b23b kind: Event lastTimestamp: "2026-06-15T06:23:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:06Z" name: success-200-isvc-6c800-predictor.18b92cf497dd303e namespace: kserve-ci-e2e-test resourceVersion: "27609" uid: 2f7fdc5b-670f-494b-9534-0e408c003386 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:22:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c800-predictor namespace: kserve-ci-e2e-test resourceVersion: "27310" uid: fb25ba57-483f-4113-9a39-2eaaa229b23b kind: Event lastTimestamp: "2026-06-15T06:23:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:36Z" name: success-200-isvc-6c800-predictor.18b92cf8168cbf07 namespace: kserve-ci-e2e-test resourceVersion: "27942" uid: 8cd645b6-8d8a-4b9e-b51d-500a5f2aadf0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:22:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c800-predictor namespace: kserve-ci-e2e-test resourceVersion: "27310" uid: fb25ba57-483f-4113-9a39-2eaaa229b23b kind: Event lastTimestamp: "2026-06-15T06:23:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:36Z" name: success-200-isvc-6c800-predictor.18b92cf816d824df namespace: kserve-ci-e2e-test resourceVersion: "27943" uid: 70fd700e-7695-415a-9668-2ed583b838a4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6c800 namespace: kserve-ci-e2e-test resourceVersion: "26923" uid: 789325de-2966-4274-84f1-554c35b83e53 kind: Event lastTimestamp: "2026-06-15T06:23:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:23:10Z" name: success-200-isvc-6c800.18b92cf118d7a981 namespace: kserve-ci-e2e-test resourceVersion: "27649" uid: 71c78347-2316-4792-88a2-e89fa7c219f9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6c800 namespace: kserve-ci-e2e-test resourceVersion: "26923" uid: 789325de-2966-4274-84f1-554c35b83e53 kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: 'Failed to update status for InferenceService "success-200-isvc-6c800": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6c800": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:22:21Z" name: success-200-isvc-6c800.18b92cf1193b54e6 namespace: kserve-ci-e2e-test resourceVersion: "26950" uid: 74fb8963-bcbe-45f8-8221-5293bc5786dc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:22:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6c800 namespace: kserve-ci-e2e-test resourceVersion: "26923" uid: 789325de-2966-4274-84f1-554c35b83e53 kind: Event lastTimestamp: "2026-06-15T06:22:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6c800": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:22:21Z" name: success-200-isvc-6c800.18b92cf1193bb048 namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 4f9001d1-4102-4612-8a70-746229a81696 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6c800 namespace: kserve-ci-e2e-test resourceVersion: "27647" uid: 789325de-2966-4274-84f1-554c35b83e53 kind: Event lastTimestamp: "2026-06-15T06:23:10Z" message: InferenceService [success-200-isvc-6c800] is Ready metadata: creationTimestamp: "2026-06-15T06:23:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:23:10Z" name: success-200-isvc-6c800.18b92cfc73422985 namespace: kserve-ci-e2e-test resourceVersion: "27648" uid: 683b783b-977a-49ac-ae7e-30b30a0a9d7a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:11:00.706048Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw namespace: kserve-ci-e2e-test resourceVersion: "20818" uid: 09814fd3-7783-461d-bcfd-b985deffc25b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-78ee8-predictor-79b588b5d-jhzfw to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:11:00Z" name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw.18b92c528f5f1e81 namespace: kserve-ci-e2e-test resourceVersion: "20826" uid: fdec9823-d028-40d8-a500-cb8f3165008c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 09814fd3-7783-461d-bcfd-b985deffc25b kind: Event lastTimestamp: "2026-06-15T06:11:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-78ee8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:00Z" name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw.18b92c529df1bd6c namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: 82f441d9-f6b9-4d65-9735-7b9cd8b2cc03 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 09814fd3-7783-461d-bcfd-b985deffc25b kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:11:01Z" name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw.18b92c52cd59460e namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: e11664d5-48ee-49fc-9408-bf24e59f6f6e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 09814fd3-7783-461d-bcfd-b985deffc25b kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw.18b92c52ceeb49f1 namespace: kserve-ci-e2e-test resourceVersion: "20891" uid: 3e1c70d3-e22a-43e6-88a4-2bf337ca4044 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 09814fd3-7783-461d-bcfd-b985deffc25b kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw.18b92c52d20ef8cb namespace: kserve-ci-e2e-test resourceVersion: "20893" uid: 2a2db0a2-59b0-4d6c-b381-250a48c9f226 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 09814fd3-7783-461d-bcfd-b985deffc25b kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw.18b92c52d2867949 namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: 351590a8-2f38-4d5e-b160-931f82dd8e2b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 09814fd3-7783-461d-bcfd-b985deffc25b kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw.18b92c52d2914dc7 namespace: kserve-ci-e2e-test resourceVersion: "20896" uid: e3462185-9e1a-4553-a261-b38b332f58b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 09814fd3-7783-461d-bcfd-b985deffc25b kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw.18b92c52d575b706 namespace: kserve-ci-e2e-test resourceVersion: "20898" uid: 433502ec-d68f-4dcb-ae7d-60bddc687b56 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 09814fd3-7783-461d-bcfd-b985deffc25b kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw.18b92c52d5efe9e9 namespace: kserve-ci-e2e-test resourceVersion: "20899" uid: a70d5047-e8aa-49e6-80cf-a520953c0027 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 09814fd3-7783-461d-bcfd-b985deffc25b kind: Event lastTimestamp: "2026-06-15T06:11:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:38Z" name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw.18b92c52e71c3f6a namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: a59c11c4-e90e-4629-874b-bf7e32f835a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 09814fd3-7783-461d-bcfd-b985deffc25b kind: Event lastTimestamp: "2026-06-15T06:12:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:10Z" name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw.18b92c62e8efecd0 namespace: kserve-ci-e2e-test resourceVersion: "21704" uid: 713290fe-be6b-4669-82e7-2c754bdc3e54 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 09814fd3-7783-461d-bcfd-b985deffc25b kind: Event lastTimestamp: "2026-06-15T06:12:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:10Z" name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw.18b92c62e8f07c5a namespace: kserve-ci-e2e-test resourceVersion: "21708" uid: 780dfcc9-1fa0-4aa0-90af-333e74fb2c64 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 09814fd3-7783-461d-bcfd-b985deffc25b kind: Event lastTimestamp: "2026-06-15T06:12:13Z" message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp 10.133.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:13Z" name: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw.18b92c636f5610af namespace: kserve-ci-e2e-test resourceVersion: "21810" uid: 9c05a36c-2abb-401c-9169-3130432d776b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-78ee8-predictor-79b588b5d namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: c8a1da9c-fede-4a62-af68-1db7ad411dcd kind: Event lastTimestamp: "2026-06-15T06:11:00Z" message: 'Created pod: success-200-isvc-78ee8-predictor-79b588b5d-jhzfw' metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:00Z" name: success-200-isvc-78ee8-predictor-79b588b5d.18b92c528ee0b479 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: 8efc96ee-3564-41e5-b6b9-131c65622e73 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-78ee8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: c1f95d9f-c6c7-4d3b-b3d9-76e666a4d610 kind: Event lastTimestamp: "2026-06-15T06:11:00Z" message: Scaled up replica set success-200-isvc-78ee8-predictor-79b588b5d from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:00Z" name: success-200-isvc-78ee8-predictor.18b92c528a6d96ee namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: 8894f7bf-6c58-42aa-a192-47e032d3ba5c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:11:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78ee8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: 21cc8a01-0b03-4973-b41b-97d3232ad5a1 kind: Event lastTimestamp: "2026-06-15T06:11:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:45Z" name: success-200-isvc-78ee8-predictor.18b92c56091eece8 namespace: kserve-ci-e2e-test resourceVersion: "21258" uid: 6a548864-49e6-4bdd-a61e-ef09114ddb5b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:11:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78ee8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: 21cc8a01-0b03-4973-b41b-97d3232ad5a1 kind: Event lastTimestamp: "2026-06-15T06:11:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:11:45Z" name: success-200-isvc-78ee8-predictor.18b92c5609904c15 namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: b63346da-8f91-49b3-bdf6-0f3572efb42b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78ee8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: 21cc8a01-0b03-4973-b41b-97d3232ad5a1 kind: Event lastTimestamp: "2026-06-15T06:12:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:00Z" name: success-200-isvc-78ee8-predictor.18b92c6086e5ac22 namespace: kserve-ci-e2e-test resourceVersion: "21535" uid: 2b80ca07-6458-4ad8-b1b5-0f4a4860785c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78ee8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: 21cc8a01-0b03-4973-b41b-97d3232ad5a1 kind: Event lastTimestamp: "2026-06-15T06:12:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:00Z" name: success-200-isvc-78ee8-predictor.18b92c608746c126 namespace: kserve-ci-e2e-test resourceVersion: "21539" uid: b2d6507f-c7d2-4975-8c7a-3ff862f327f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78ee8 namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: 1a8bd2c0-12bf-480d-833a-6056b8b402f9 kind: Event lastTimestamp: "2026-06-15T06:11:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:48Z" name: success-200-isvc-78ee8.18b92c528a6b871c namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 79c4f8a0-efc8-4e60-9a9f-357435e15a82 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78ee8 namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: 1a8bd2c0-12bf-480d-833a-6056b8b402f9 kind: Event lastTimestamp: "2026-06-15T06:11:00Z" message: 'Failed to update status for InferenceService "success-200-isvc-78ee8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-78ee8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:00Z" name: success-200-isvc-78ee8.18b92c528b0c9ca3 namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: e0c35969-0041-48a9-a3ca-0057668292f6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78ee8 namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: 1a8bd2c0-12bf-480d-833a-6056b8b402f9 kind: Event lastTimestamp: "2026-06-15T06:11:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-78ee8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:00Z" name: success-200-isvc-78ee8.18b92c528b0cdb50 namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: da3c566a-6ff2-439a-beb8-762834f1b862 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78ee8 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: 1a8bd2c0-12bf-480d-833a-6056b8b402f9 kind: Event lastTimestamp: "2026-06-15T06:11:48Z" message: InferenceService [success-200-isvc-78ee8] is Ready metadata: creationTimestamp: "2026-06-15T06:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:11:48Z" name: success-200-isvc-78ee8.18b92c5da303bfb1 namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: 8d109dea-92b2-4609-abad-c3fe7b586fdd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:12:10.939983Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: 1cf6bf62-6c07-4b14-bbcd-f43cb0c4d046 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:12:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:12:10Z" name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n.18b92c62e9a3d97f namespace: kserve-ci-e2e-test resourceVersion: "21710" uid: 3135f4fa-31e4-4fcd-bf5d-721327559236 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n namespace: kserve-ci-e2e-test resourceVersion: "21717" uid: 1cf6bf62-6c07-4b14-bbcd-f43cb0c4d046 kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:12:11Z" name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n.18b92c6303edb9fa namespace: kserve-ci-e2e-test resourceVersion: "21769" uid: e3c61b2f-a34c-4e45-97b1-e2ff99dcd7fc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 1cf6bf62-6c07-4b14-bbcd-f43cb0c4d046 kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:11Z" name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n.18b92c630622caf2 namespace: kserve-ci-e2e-test resourceVersion: "21772" uid: a611b968-629a-444d-8126-0cf871979c9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 1cf6bf62-6c07-4b14-bbcd-f43cb0c4d046 kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:11Z" name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n.18b92c6308d8ae46 namespace: kserve-ci-e2e-test resourceVersion: "21778" uid: c49c526e-2453-4808-858d-ecadefcf2cab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 1cf6bf62-6c07-4b14-bbcd-f43cb0c4d046 kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:11Z" name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n.18b92c630959f615 namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: 15d299b7-aeb1-415d-bdfb-ce971d9e8529 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 1cf6bf62-6c07-4b14-bbcd-f43cb0c4d046 kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:11Z" name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n.18b92c63096589cb namespace: kserve-ci-e2e-test resourceVersion: "21781" uid: 178a850c-b616-4bf5-a666-2f50c4546e3d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 1cf6bf62-6c07-4b14-bbcd-f43cb0c4d046 kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:11Z" name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n.18b92c630cc13e1b namespace: kserve-ci-e2e-test resourceVersion: "21782" uid: 4d6b9c3b-4cfd-454c-bfca-69dbc7bbe243 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 1cf6bf62-6c07-4b14-bbcd-f43cb0c4d046 kind: Event lastTimestamp: "2026-06-15T06:12:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:11Z" name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n.18b92c630d52a440 namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: 7c3a15ac-0652-4f8d-a9f9-8fdb3348a827 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 1cf6bf62-6c07-4b14-bbcd-f43cb0c4d046 kind: Event lastTimestamp: "2026-06-15T06:12:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:12:49Z" name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n.18b92c63838bb96e namespace: kserve-ci-e2e-test resourceVersion: "22135" uid: 7f076554-a6bb-43fc-a25d-ca29f3ab8f33 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 1cf6bf62-6c07-4b14-bbcd-f43cb0c4d046 kind: Event lastTimestamp: "2026-06-15T06:21:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:25Z" name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n.18b92ce41a5a6f7f namespace: kserve-ci-e2e-test resourceVersion: "26365" uid: 5dca0e48-a27c-49a9-96cf-0afe30228a32 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:21:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 1cf6bf62-6c07-4b14-bbcd-f43cb0c4d046 kind: Event lastTimestamp: "2026-06-15T06:21:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:21:25Z" name: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n.18b92ce41a5b03a9 namespace: kserve-ci-e2e-test resourceVersion: "26368" uid: 8363b3a1-f4d8-4e51-afd5-10d591920679 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b50b1-predictor-d45d95b7b namespace: kserve-ci-e2e-test resourceVersion: "21689" uid: c274a70c-91a4-4457-841a-5ffd9f0342bc kind: Event lastTimestamp: "2026-06-15T06:12:10Z" message: 'Created pod: success-200-isvc-b50b1-predictor-d45d95b7b-nkv5n' metadata: creationTimestamp: "2026-06-15T06:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:10Z" name: success-200-isvc-b50b1-predictor-d45d95b7b.18b92c62e92474c8 namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 628c5d2b-7cce-4bd8-a79b-4c4f44252756 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b50b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21681" uid: 6b2c5d68-8ee0-49cd-a28f-d13bdd46fc7a kind: Event lastTimestamp: "2026-06-15T06:12:10Z" message: Scaled up replica set success-200-isvc-b50b1-predictor-d45d95b7b from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:10Z" name: success-200-isvc-b50b1-predictor.18b92c62e76f140d namespace: kserve-ci-e2e-test resourceVersion: "21694" uid: 3eeefa7b-e0c5-40a0-950a-462e1172a934 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:12:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b50b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21688" uid: f54c28ea-bcc4-4572-98a4-4121ec44f10d kind: Event lastTimestamp: "2026-06-15T06:12:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:12:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:40Z" name: success-200-isvc-b50b1-predictor.18b92c666649ee5d namespace: kserve-ci-e2e-test resourceVersion: "22048" uid: 930eadb3-661c-4e83-b144-536716f03a8f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:12:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b50b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21688" uid: f54c28ea-bcc4-4572-98a4-4121ec44f10d kind: Event lastTimestamp: "2026-06-15T06:12:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:12:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:12:40Z" name: success-200-isvc-b50b1-predictor.18b92c66669d47d4 namespace: kserve-ci-e2e-test resourceVersion: "22049" uid: 2ac21ea3-2585-48b3-8a07-9ab35baa6e56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:12:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b50b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21922" uid: f54c28ea-bcc4-4572-98a4-4121ec44f10d kind: Event lastTimestamp: "2026-06-15T06:13:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:25Z" name: success-200-isvc-b50b1-predictor.18b92c6d64c2cc52 namespace: kserve-ci-e2e-test resourceVersion: "22668" uid: 246224b3-9053-4098-96e2-f50e42eb3ce6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-15T06:12:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b50b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21922" uid: f54c28ea-bcc4-4572-98a4-4121ec44f10d kind: Event lastTimestamp: "2026-06-15T06:13:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:13:26Z" name: success-200-isvc-b50b1-predictor.18b92c6d652faf68 namespace: kserve-ci-e2e-test resourceVersion: "22670" uid: 21f3152b-e6ee-48c2-bded-2fb27fb85e18 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:12:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: e70d0440-ce4b-4e0c-a987-d0c03a11da8b kind: Event lastTimestamp: "2026-06-15T06:12:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:59Z" name: success-200-isvc-b50b1.18b92c62e741e2a1 namespace: kserve-ci-e2e-test resourceVersion: "22266" uid: b22d5eaf-ea0b-4a43-a2bd-aab7570a69a9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: e70d0440-ce4b-4e0c-a987-d0c03a11da8b kind: Event lastTimestamp: "2026-06-15T06:12:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-b50b1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b50b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:10Z" name: success-200-isvc-b50b1.18b92c62e7f4a0a8 namespace: kserve-ci-e2e-test resourceVersion: "21699" uid: b6cb62a2-c433-4970-b51b-f51f42343379 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: e70d0440-ce4b-4e0c-a987-d0c03a11da8b kind: Event lastTimestamp: "2026-06-15T06:12:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b50b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:10Z" name: success-200-isvc-b50b1.18b92c62e7f5072b namespace: kserve-ci-e2e-test resourceVersion: "21700" uid: 2d20f19d-4b8a-4875-9136-9583db223569 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:12:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b50b1 namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: e70d0440-ce4b-4e0c-a987-d0c03a11da8b kind: Event lastTimestamp: "2026-06-15T06:12:59Z" message: InferenceService [success-200-isvc-b50b1] is Ready metadata: creationTimestamp: "2026-06-15T06:12:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:12:59Z" name: success-200-isvc-b50b1.18b92c6e3e4b4b98 namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: 99c84648-89f9-455b-887d-b8e0ed4bdfc8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:01:45.626299Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 1e314429-bd2e-44b9-9c6e-f4fe2fbc0c82 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:01:45Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75.18b92bd1520a4efb namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 280e5678-5266-42bd-ae02-a90bbbdc3dea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 namespace: kserve-ci-e2e-test resourceVersion: "16472" uid: 1e314429-bd2e-44b9-9c6e-f4fe2fbc0c82 kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:01:46Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75.18b92bd16d24d5d7 namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 12248ed1-2618-49be-8bd0-0e7f5da5e540 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 1e314429-bd2e-44b9-9c6e-f4fe2fbc0c82 kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1619" already present on machine metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:46Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75.18b92bd16e8cf037 namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 9ef1bda7-3802-42f3-b161-39f7c92d5580 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 1e314429-bd2e-44b9-9c6e-f4fe2fbc0c82 kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:46Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75.18b92bd1714f7243 namespace: kserve-ci-e2e-test resourceVersion: "16512" uid: 0ef60a77-671d-4751-9613-1a0bf56819d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 1e314429-bd2e-44b9-9c6e-f4fe2fbc0c82 kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:46Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75.18b92bd171c1d992 namespace: kserve-ci-e2e-test resourceVersion: "16513" uid: 76210115-89ec-404a-8bd9-68c4dd178fe3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 1e314429-bd2e-44b9-9c6e-f4fe2fbc0c82 kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:46Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75.18b92bd171cd08a7 namespace: kserve-ci-e2e-test resourceVersion: "16514" uid: b5bdc673-9957-480e-82b8-ff70875f8d88 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 1e314429-bd2e-44b9-9c6e-f4fe2fbc0c82 kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:46Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75.18b92bd1750ce7a4 namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: ed893148-02bb-4fa6-af07-1db646394598 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 1e314429-bd2e-44b9-9c6e-f4fe2fbc0c82 kind: Event lastTimestamp: "2026-06-15T06:01:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:46Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75.18b92bd175780d9f namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: a9f69a6a-9a39-4485-8074-c20c45815fb6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-06-15T06:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 1e314429-bd2e-44b9-9c6e-f4fe2fbc0c82 kind: Event lastTimestamp: "2026-06-15T06:11:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:03Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75.18b92bd1ce363713 namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: 1cbdec47-e4e4-4f67-bd02-f3246ba9a498 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 1e314429-bd2e-44b9-9c6e-f4fe2fbc0c82 kind: Event lastTimestamp: "2026-06-15T06:11:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ea4bd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:04Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75.18b92c528bd8b168 namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: 525854f5-0d52-4526-aed8-fab27169080e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 1e314429-bd2e-44b9-9c6e-f4fe2fbc0c82 kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75.18b92c52aae1404d namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: fa9a6e1c-fae5-499d-bc99-4ee8ad593bb7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 1e314429-bd2e-44b9-9c6e-f4fe2fbc0c82 kind: Event lastTimestamp: "2026-06-15T06:11:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-15T06:11:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:01Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75.18b92c52aae1a393 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 3ff68dae-a77a-482c-96ad-d53a690879bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 1e314429-bd2e-44b9-9c6e-f4fe2fbc0c82 kind: Event lastTimestamp: "2026-06-15T06:11:03Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-15T06:11:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:03Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75.18b92c5342cb0b59 namespace: kserve-ci-e2e-test resourceVersion: "20933" uid: e79c676b-88dd-4fbf-8429-d98adb1071bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ea4bd-predictor-65c4fc694d namespace: kserve-ci-e2e-test resourceVersion: "16435" uid: 87b5a0d9-a6bf-4a64-80f8-333609a87e0d kind: Event lastTimestamp: "2026-06-15T06:01:45Z" message: 'Created pod: success-200-isvc-ea4bd-predictor-65c4fc694d-sbg75' metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:45Z" name: success-200-isvc-ea4bd-predictor-65c4fc694d.18b92bd1519310e9 namespace: kserve-ci-e2e-test resourceVersion: "16450" uid: b7339182-3d1e-4249-872b-1a80f26f3be4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ea4bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16432" uid: 509478e4-82c5-410e-b1e3-987b53360ba1 kind: Event lastTimestamp: "2026-06-15T06:01:45Z" message: Scaled up replica set success-200-isvc-ea4bd-predictor-65c4fc694d from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:45Z" name: success-200-isvc-ea4bd-predictor.18b92bd14ea90fff namespace: kserve-ci-e2e-test resourceVersion: "16438" uid: 67232cd2-50ad-45e8-a33e-fc7114cb513a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ea4bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 069c9b70-e787-4df7-87cd-c9e36e3d8ae9 kind: Event lastTimestamp: "2026-06-15T06:02:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:00Z" name: success-200-isvc-ea4bd-predictor.18b92bd4cd5bba88 namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: b9595b49-5052-429a-87c2-86d316c5d0cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ea4bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 069c9b70-e787-4df7-87cd-c9e36e3d8ae9 kind: Event lastTimestamp: "2026-06-15T06:02:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:00Z" name: success-200-isvc-ea4bd-predictor.18b92bd4cdfae1a4 namespace: kserve-ci-e2e-test resourceVersion: "16663" uid: 3c7ef3f9-b701-45fe-bb8e-9e43dd16dda5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:02:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ea4bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: 069c9b70-e787-4df7-87cd-c9e36e3d8ae9 kind: Event lastTimestamp: "2026-06-15T06:03:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:00Z" name: success-200-isvc-ea4bd-predictor.18b92bd84cb0585f namespace: kserve-ci-e2e-test resourceVersion: "17418" uid: 7ce7ae18-27c4-438a-bcd9-57b75c69d061 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:02:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ea4bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: 069c9b70-e787-4df7-87cd-c9e36e3d8ae9 kind: Event lastTimestamp: "2026-06-15T06:03:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:00Z" name: success-200-isvc-ea4bd-predictor.18b92bd84d227663 namespace: kserve-ci-e2e-test resourceVersion: "17419" uid: d3ae5704-b5d1-4c0c-b84d-914ad1b00a08 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: 050eefe1-3ba0-44df-801d-4b725636974b kind: Event lastTimestamp: "2026-06-15T06:02:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:33Z" name: success-200-isvc-ea4bd.18b92bd14e8072af namespace: kserve-ci-e2e-test resourceVersion: "16991" uid: c922f38c-3f40-439a-b612-be9920808471 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: 050eefe1-3ba0-44df-801d-4b725636974b kind: Event lastTimestamp: "2026-06-15T06:01:45Z" message: 'Failed to update status for InferenceService "success-200-isvc-ea4bd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ea4bd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:45Z" name: success-200-isvc-ea4bd.18b92bd14ef3e738 namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: fa117646-f5e1-4478-b500-663a19a4fc82 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: 050eefe1-3ba0-44df-801d-4b725636974b kind: Event lastTimestamp: "2026-06-15T06:01:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ea4bd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:45Z" name: success-200-isvc-ea4bd.18b92bd14ef44b5d namespace: kserve-ci-e2e-test resourceVersion: "16443" uid: d2f5544a-bb04-466c-a560-257d5b7ea9b1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "16989" uid: 050eefe1-3ba0-44df-801d-4b725636974b kind: Event lastTimestamp: "2026-06-15T06:02:33Z" message: InferenceService [success-200-isvc-ea4bd] is Ready metadata: creationTimestamp: "2026-06-15T06:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:33Z" name: success-200-isvc-ea4bd.18b92bdc897d13e7 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: 38a133ad-7bf7-4a87-8230-8e1933d264cf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:01:31.274185Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5cf27-599c8856d9-mrll6 namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: 5c92c82f-7aa6-468a-b3bc-d0a338a99334 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-5cf27-599c8856d9-mrll6 to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:01:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:01:31Z" name: switch-graph-5cf27-599c8856d9-mrll6.18b92bcdfa965ffc namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: 17760239-e6a8-41a0-968b-0c009eace1bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:31Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5cf27-599c8856d9-mrll6 namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: 5c92c82f-7aa6-468a-b3bc-d0a338a99334 kind: Event lastTimestamp: "2026-06-15T06:01:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-5cf27-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:01:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:31Z" name: switch-graph-5cf27-599c8856d9-mrll6.18b92bce06774b47 namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: a1157b1b-a002-4ef5-ba8c-9ec257133317 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:32Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5cf27-599c8856d9-mrll6 namespace: kserve-ci-e2e-test resourceVersion: "16248" uid: 5c92c82f-7aa6-468a-b3bc-d0a338a99334 kind: Event lastTimestamp: "2026-06-15T06:01:32Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:01:32Z" name: switch-graph-5cf27-599c8856d9-mrll6.18b92bce38199b91 namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: 56c95e42-a77e-4268-8887-5d8a9616ece2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5cf27} kind: Pod name: switch-graph-5cf27-599c8856d9-mrll6 namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: 5c92c82f-7aa6-468a-b3bc-d0a338a99334 kind: Event lastTimestamp: "2026-06-15T06:01:32Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" metadata: creationTimestamp: "2026-06-15T06:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:32Z" name: switch-graph-5cf27-599c8856d9-mrll6.18b92bce397cbbf0 namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: 94fd575f-666f-4774-9b37-7d5b7c197729 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5cf27} kind: Pod name: switch-graph-5cf27-599c8856d9-mrll6 namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: 5c92c82f-7aa6-468a-b3bc-d0a338a99334 kind: Event lastTimestamp: "2026-06-15T06:01:34Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" in 2.491s (2.491s including waiting). Image size: 218065501 bytes.' metadata: creationTimestamp: "2026-06-15T06:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:34Z" name: switch-graph-5cf27-599c8856d9-mrll6.18b92bcece021687 namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 49ac590f-36a7-4377-bce0-9a7bf78d5163 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5cf27} kind: Pod name: switch-graph-5cf27-599c8856d9-mrll6 namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: 5c92c82f-7aa6-468a-b3bc-d0a338a99334 kind: Event lastTimestamp: "2026-06-15T06:01:34Z" message: 'Created container: switch-graph-5cf27' metadata: creationTimestamp: "2026-06-15T06:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:34Z" name: switch-graph-5cf27-599c8856d9-mrll6.18b92bced0f43d64 namespace: kserve-ci-e2e-test resourceVersion: "16285" uid: a545d4b8-f962-452f-b509-a33fa08ff002 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5cf27} kind: Pod name: switch-graph-5cf27-599c8856d9-mrll6 namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: 5c92c82f-7aa6-468a-b3bc-d0a338a99334 kind: Event lastTimestamp: "2026-06-15T06:01:34Z" message: Started container switch-graph-5cf27 metadata: creationTimestamp: "2026-06-15T06:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:34Z" name: switch-graph-5cf27-599c8856d9-mrll6.18b92bced15cb6b4 namespace: kserve-ci-e2e-test resourceVersion: "16286" uid: 9ae6e797-e980-4622-9f9b-1f5692998f4c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5cf27} kind: Pod name: switch-graph-5cf27-599c8856d9-mrll6 namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: 5c92c82f-7aa6-468a-b3bc-d0a338a99334 kind: Event lastTimestamp: "2026-06-15T06:01:45Z" message: Stopping container switch-graph-5cf27 metadata: creationTimestamp: "2026-06-15T06:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:01:45Z" name: switch-graph-5cf27-599c8856d9-mrll6.18b92bd146d4c6a5 namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: 47265732-fe6e-4b43-ab6c-555d21df8b4b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5cf27} kind: Pod name: switch-graph-5cf27-599c8856d9-mrll6 namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: 5c92c82f-7aa6-468a-b3bc-d0a338a99334 kind: Event lastTimestamp: "2026-06-15T06:02:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:11Z" name: switch-graph-5cf27-599c8856d9-mrll6.18b92bd18fcd100c namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: 7fbe8222-dedb-4572-883a-9b7b000ac786 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-5cf27-599c8856d9 namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 8e3104bb-1b67-44b7-9122-2a72c3320011 kind: Event lastTimestamp: "2026-06-15T06:01:31Z" message: 'Created pod: switch-graph-5cf27-599c8856d9-mrll6' metadata: creationTimestamp: "2026-06-15T06:01:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:31Z" name: switch-graph-5cf27-599c8856d9.18b92bcdfa242091 namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: 4283c572-8cd5-47d4-adf7-31bd7f4ea83a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-5cf27 namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 8630e6fe-be7a-4903-b674-eb40100c8dba kind: Event lastTimestamp: "2026-06-15T06:01:31Z" message: Scaled up replica set switch-graph-5cf27-599c8856d9 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:01:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:01:31Z" name: switch-graph-5cf27.18b92bcdf93700a8 namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 2b3aa695-51e2-46aa-92ea-6b0876b9b5a7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:01:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-5cf27 namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: 4ab0cd06-3382-4964-bcc2-75028a95b5f0 kind: Event lastTimestamp: "2026-06-15T06:01:31Z" message: InferenceGraph [switch-graph-5cf27] is Ready metadata: creationTimestamp: "2026-06-15T06:01:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:01:31Z" name: switch-graph-5cf27.18b92bcdfe1dd8e2 namespace: kserve-ci-e2e-test resourceVersion: "16252" uid: 5f47b465-1fa7-479a-b7a2-22c4964c2100 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:23:21.720933Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6c800-596ccdd7bb-g54r6 namespace: kserve-ci-e2e-test resourceVersion: "27759" uid: 54cd1c28-8bd5-4369-9d34-ebfe96c2a165 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-6c800-596ccdd7bb-g54r6 to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:23:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:23:21Z" name: switch-graph-6c800-596ccdd7bb-g54r6.18b92cff174ce1d7 namespace: kserve-ci-e2e-test resourceVersion: "27764" uid: b7c553a7-fce8-4405-808e-7ca082bffec0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-15T06:23:21Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6c800-596ccdd7bb-g54r6 namespace: kserve-ci-e2e-test resourceVersion: "27761" uid: 54cd1c28-8bd5-4369-9d34-ebfe96c2a165 kind: Event lastTimestamp: "2026-06-15T06:23:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-6c800-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:23:22Z" name: switch-graph-6c800-596ccdd7bb-g54r6.18b92cff23dd23cd namespace: kserve-ci-e2e-test resourceVersion: "27780" uid: 38dfa9fb-a3e0-43d3-8ce0-23bdc1be2e17 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:23Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-6c800-596ccdd7bb-g54r6 namespace: kserve-ci-e2e-test resourceVersion: "27768" uid: 54cd1c28-8bd5-4369-9d34-ebfe96c2a165 kind: Event lastTimestamp: "2026-06-15T06:23:23Z" message: Add eth0 [10.133.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:23:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:23:23Z" name: switch-graph-6c800-596ccdd7bb-g54r6.18b92cff8ae90f6b namespace: kserve-ci-e2e-test resourceVersion: "27795" uid: 43182512-12ec-47c0-9afa-7f8246380385 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6c800} kind: Pod name: switch-graph-6c800-596ccdd7bb-g54r6 namespace: kserve-ci-e2e-test resourceVersion: "27761" uid: 54cd1c28-8bd5-4369-9d34-ebfe96c2a165 kind: Event lastTimestamp: "2026-06-15T06:23:23Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" already present on machine metadata: creationTimestamp: "2026-06-15T06:23:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:23:23Z" name: switch-graph-6c800-596ccdd7bb-g54r6.18b92cff8c310038 namespace: kserve-ci-e2e-test resourceVersion: "27797" uid: 86a37a19-6038-4989-a16e-0f0988d591b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6c800} kind: Pod name: switch-graph-6c800-596ccdd7bb-g54r6 namespace: kserve-ci-e2e-test resourceVersion: "27761" uid: 54cd1c28-8bd5-4369-9d34-ebfe96c2a165 kind: Event lastTimestamp: "2026-06-15T06:23:23Z" message: 'Created container: switch-graph-6c800' metadata: creationTimestamp: "2026-06-15T06:23:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:23:23Z" name: switch-graph-6c800-596ccdd7bb-g54r6.18b92cff9013dbd5 namespace: kserve-ci-e2e-test resourceVersion: "27798" uid: c6c7f913-c62a-4cc8-84ec-bde1b51ded9e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6c800} kind: Pod name: switch-graph-6c800-596ccdd7bb-g54r6 namespace: kserve-ci-e2e-test resourceVersion: "27761" uid: 54cd1c28-8bd5-4369-9d34-ebfe96c2a165 kind: Event lastTimestamp: "2026-06-15T06:23:23Z" message: Started container switch-graph-6c800 metadata: creationTimestamp: "2026-06-15T06:23:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:23:23Z" name: switch-graph-6c800-596ccdd7bb-g54r6.18b92cff908d2ff0 namespace: kserve-ci-e2e-test resourceVersion: "27799" uid: 60bf5933-7ed3-45b7-82be-dabc9dd16d50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:39:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6c800} kind: Pod name: switch-graph-6c800-596ccdd7bb-g54r6 namespace: kserve-ci-e2e-test resourceVersion: "27761" uid: 54cd1c28-8bd5-4369-9d34-ebfe96c2a165 kind: Event lastTimestamp: "2026-06-15T06:39:41Z" message: Stopping container switch-graph-6c800 metadata: creationTimestamp: "2026-06-15T06:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:39:41Z" name: switch-graph-6c800-596ccdd7bb-g54r6.18b92de321e620c8 namespace: kserve-ci-e2e-test resourceVersion: "34643" uid: 8a13092a-ec1e-4f06-bd18-0ba2d43202ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-15T06:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-6c800} kind: Pod name: switch-graph-6c800-596ccdd7bb-g54r6 namespace: kserve-ci-e2e-test resourceVersion: "27761" uid: 54cd1c28-8bd5-4369-9d34-ebfe96c2a165 kind: Event lastTimestamp: "2026-06-15T06:40:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:40:05Z" name: switch-graph-6c800-596ccdd7bb-g54r6.18b92de429b94eef namespace: kserve-ci-e2e-test resourceVersion: "34888" uid: 480efb4e-f2c1-48c8-b364-8db7e0089b16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-6c800-596ccdd7bb namespace: kserve-ci-e2e-test resourceVersion: "27756" uid: f0651b96-7797-4d8f-aa70-335145f0efb8 kind: Event lastTimestamp: "2026-06-15T06:23:21Z" message: 'Created pod: switch-graph-6c800-596ccdd7bb-g54r6' metadata: creationTimestamp: "2026-06-15T06:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:21Z" name: switch-graph-6c800-596ccdd7bb.18b92cff16e4e0d0 namespace: kserve-ci-e2e-test resourceVersion: "27760" uid: 337dd2c5-6564-4880-a465-67210553cc22 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-6c800 namespace: kserve-ci-e2e-test resourceVersion: "27753" uid: 5a505893-7902-4b30-a2f0-74852fd3c25b kind: Event lastTimestamp: "2026-06-15T06:23:21Z" message: Scaled up replica set switch-graph-6c800-596ccdd7bb from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:23:21Z" name: switch-graph-6c800.18b92cff16148286 namespace: kserve-ci-e2e-test resourceVersion: "27757" uid: 42fdc649-fba3-4105-b22a-e9fdb5f7cfd4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:23:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-6c800 namespace: kserve-ci-e2e-test resourceVersion: "27773" uid: 3c15b776-00bc-44d3-8068-59b184e3784d kind: Event lastTimestamp: "2026-06-15T06:23:21Z" message: InferenceGraph [switch-graph-6c800] is Ready metadata: creationTimestamp: "2026-06-15T06:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:23:21Z" name: switch-graph-6c800.18b92cff1af9e718 namespace: kserve-ci-e2e-test resourceVersion: "27775" uid: 099a0d51-2297-47a7-8dd1-6d4f74d31c07 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:23:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6c800 namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 1a945896-9e4e-417f-8f49-631d6f7b9184 kind: Event lastTimestamp: "2026-06-15T06:24:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:24:21Z" name: switch-graph-6c800.18b92d02947db99b namespace: kserve-ci-e2e-test resourceVersion: "28287" uid: acc5fe56-f0db-4db2-a445-566c520a21be reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-15T06:23:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-6c800 namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 1a945896-9e4e-417f-8f49-631d6f7b9184 kind: Event lastTimestamp: "2026-06-15T06:24:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:24:21Z" name: switch-graph-6c800.18b92d0294d3e700 namespace: kserve-ci-e2e-test resourceVersion: "28288" uid: 0b0ca9ea-cd66-4f66-9119-f87be61bcc55 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-06-15T06:02:45.671679Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ea4bd-76d9f94847-lx78n namespace: kserve-ci-e2e-test resourceVersion: "17256" uid: 8adc762f-ac90-4697-a69a-70447f4c4eeb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-ea4bd-76d9f94847-lx78n to ip-10-0-133-111.ec2.internal metadata: creationTimestamp: "2026-06-15T06:02:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-15T06:02:45Z" name: switch-graph-ea4bd-76d9f94847-lx78n.18b92bdf4d06123f namespace: kserve-ci-e2e-test resourceVersion: "17261" uid: 1e64ecdb-d0d0-4fd9-b682-847277e94923 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-59fc9d4647-fvrv9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:45Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ea4bd-76d9f94847-lx78n namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: 8adc762f-ac90-4697-a69a-70447f4c4eeb kind: Event lastTimestamp: "2026-06-15T06:02:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-ea4bd-serving-cert" not found' metadata: creationTimestamp: "2026-06-15T06:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:45Z" name: switch-graph-ea4bd-76d9f94847-lx78n.18b92bdf53f8bbc5 namespace: kserve-ci-e2e-test resourceVersion: "17274" uid: ac431383-0254-4f80-8341-a8387e691363 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:46Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ea4bd-76d9f94847-lx78n namespace: kserve-ci-e2e-test resourceVersion: "17265" uid: 8adc762f-ac90-4697-a69a-70447f4c4eeb kind: Event lastTimestamp: "2026-06-15T06:02:46Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-15T06:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-15T06:02:46Z" name: switch-graph-ea4bd-76d9f94847-lx78n.18b92bdf8ad486b0 namespace: kserve-ci-e2e-test resourceVersion: "17288" uid: 166c1d6b-160f-47bd-9882-be93a7f0dfcc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ea4bd} kind: Pod name: switch-graph-ea4bd-76d9f94847-lx78n namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: 8adc762f-ac90-4697-a69a-70447f4c4eeb kind: Event lastTimestamp: "2026-06-15T06:02:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:c92ab94c818c88d4324be5bb5a3d9bda3e4173010dc47851ff3051c5a06f1614" already present on machine metadata: creationTimestamp: "2026-06-15T06:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:46Z" name: switch-graph-ea4bd-76d9f94847-lx78n.18b92bdf8c2a7259 namespace: kserve-ci-e2e-test resourceVersion: "17290" uid: 641636a2-9d73-4e2d-8a39-35e3354095f9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ea4bd} kind: Pod name: switch-graph-ea4bd-76d9f94847-lx78n namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: 8adc762f-ac90-4697-a69a-70447f4c4eeb kind: Event lastTimestamp: "2026-06-15T06:02:46Z" message: 'Created container: switch-graph-ea4bd' metadata: creationTimestamp: "2026-06-15T06:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:46Z" name: switch-graph-ea4bd-76d9f94847-lx78n.18b92bdf8f48a9b3 namespace: kserve-ci-e2e-test resourceVersion: "17294" uid: 38414016-caf7-415c-9c54-2806c7863534 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ea4bd} kind: Pod name: switch-graph-ea4bd-76d9f94847-lx78n namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: 8adc762f-ac90-4697-a69a-70447f4c4eeb kind: Event lastTimestamp: "2026-06-15T06:02:46Z" message: Started container switch-graph-ea4bd metadata: creationTimestamp: "2026-06-15T06:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:02:46Z" name: switch-graph-ea4bd-76d9f94847-lx78n.18b92bdf8fc37e29 namespace: kserve-ci-e2e-test resourceVersion: "17295" uid: c16b6b05-c8eb-4012-aabc-ab4b0d1d0c58 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ea4bd} kind: Pod name: switch-graph-ea4bd-76d9f94847-lx78n namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: 8adc762f-ac90-4697-a69a-70447f4c4eeb kind: Event lastTimestamp: "2026-06-15T06:11:00Z" message: Stopping container switch-graph-ea4bd metadata: creationTimestamp: "2026-06-15T06:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:00Z" name: switch-graph-ea4bd-76d9f94847-lx78n.18b92c5281c0bc23 namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: 8da83e3c-d674-4656-b305-1633f967bc54 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-15T06:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ea4bd} kind: Pod name: switch-graph-ea4bd-76d9f94847-lx78n namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: 8adc762f-ac90-4697-a69a-70447f4c4eeb kind: Event lastTimestamp: "2026-06-15T06:11:27Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-15T06:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-15T06:11:27Z" name: switch-graph-ea4bd-76d9f94847-lx78n.18b92c5317780c80 namespace: kserve-ci-e2e-test resourceVersion: "21125" uid: 5b5b2d9a-ffcd-4c82-968b-3cffd6eb65d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-111.ec2.internal source: component: kubelet host: ip-10-0-133-111.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-ea4bd-76d9f94847 namespace: kserve-ci-e2e-test resourceVersion: "17253" uid: 65714df1-5355-4f18-999f-843fa9461f45 kind: Event lastTimestamp: "2026-06-15T06:02:45Z" message: 'Created pod: switch-graph-ea4bd-76d9f94847-lx78n' metadata: creationTimestamp: "2026-06-15T06:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:45Z" name: switch-graph-ea4bd-76d9f94847.18b92bdf4c81e52d namespace: kserve-ci-e2e-test resourceVersion: "17257" uid: f85c2954-c5c9-40f6-8dec-c74be1452c0c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "17250" uid: ca019c08-03a4-4589-9f79-126b09c67047 kind: Event lastTimestamp: "2026-06-15T06:02:45Z" message: Scaled up replica set switch-graph-ea4bd-76d9f94847 from 0 to 1 metadata: creationTimestamp: "2026-06-15T06:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:02:45Z" name: switch-graph-ea4bd.18b92bdf4bd49299 namespace: kserve-ci-e2e-test resourceVersion: "17254" uid: a214396e-4b7d-4ff8-b136-1d11a31bd115 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "17271" uid: 77d67880-6de8-4701-b4b5-a6cdec8dd4b1 kind: Event lastTimestamp: "2026-06-15T06:02:45Z" message: InferenceGraph [switch-graph-ea4bd] is Ready metadata: creationTimestamp: "2026-06-15T06:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:45Z" name: switch-graph-ea4bd.18b92bdf534bbab1 namespace: kserve-ci-e2e-test resourceVersion: "17273" uid: b1ac1fe4-b86c-418e-a3c0-ffeff8db9eae reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "17244" uid: 77d67880-6de8-4701-b4b5-a6cdec8dd4b1 kind: Event lastTimestamp: "2026-06-15T06:02:45Z" message: 'Failed to update status for InferenceGraph "switch-graph-ea4bd": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-ea4bd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:45Z" name: switch-graph-ea4bd.18b92bdf5550adff namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: 15e10fd5-20db-4340-9b40-2293033087f8 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:02:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "17244" uid: 77d67880-6de8-4701-b4b5-a6cdec8dd4b1 kind: Event lastTimestamp: "2026-06-15T06:02:45Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-ea4bd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-15T06:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-15T06:02:45Z" name: switch-graph-ea4bd.18b92bdf5550fb86 namespace: kserve-ci-e2e-test resourceVersion: "17276" uid: bb420060-1133-4660-bcdf-5138873b5e72 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "17252" uid: fb0436a5-f8cf-4fb2-ad32-0e6499728c98 kind: Event lastTimestamp: "2026-06-15T06:03:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:00Z" name: switch-graph-ea4bd.18b92be2ca801e7f namespace: kserve-ci-e2e-test resourceVersion: "17420" uid: dcfdcf4c-8fc5-49c6-8d75-70479374dd0e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "17252" uid: fb0436a5-f8cf-4fb2-ad32-0e6499728c98 kind: Event lastTimestamp: "2026-06-15T06:03:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-15T06:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:00Z" name: switch-graph-ea4bd.18b92be2cae65bb1 namespace: kserve-ci-e2e-test resourceVersion: "17422" uid: aa60b0e8-61b8-4b13-8631-7b37f5aac72f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "17421" uid: fb0436a5-f8cf-4fb2-ad32-0e6499728c98 kind: Event lastTimestamp: "2026-06-15T06:03:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:15Z" name: switch-graph-ea4bd.18b92be6499a8027 namespace: kserve-ci-e2e-test resourceVersion: "17537" uid: 0c6f24f9-844a-4d1b-be83-a991c61bc93a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-15T06:03:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ea4bd namespace: kserve-ci-e2e-test resourceVersion: "17421" uid: fb0436a5-f8cf-4fb2-ad32-0e6499728c98 kind: Event lastTimestamp: "2026-06-15T06:03:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-15T06:03:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-15T06:03:15Z" name: switch-graph-ea4bd.18b92be649e128d1 namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: 78dca0d7-36bd-4822-9211-4d4d51bd7475 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34901"