--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:12:26.241509Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1ab93-5c487f9bf4-zvtmk namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 7774d6c0-f8dc-4d18-8782-5d3488aec90f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-1ab93-5c487f9bf4-zvtmk to ip-10-0-131-231.ec2.internal metadata: creationTimestamp: "2026-07-16T22:12:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:12:26Z" name: ensemble-graph-1ab93-5c487f9bf4-zvtmk.18c2e4c71dc9a593 namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 26d2ee25-b07d-4288-9eb3-212b6e99f110 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:26Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1ab93-5c487f9bf4-zvtmk namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: 7774d6c0-f8dc-4d18-8782-5d3488aec90f kind: Event lastTimestamp: "2026-07-16T22:12:26Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:12:26Z" name: ensemble-graph-1ab93-5c487f9bf4-zvtmk.18c2e4c73736dbbd namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 44b7241a-3da0-499b-8f95-fb2e0d9819c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1ab93} kind: Pod name: ensemble-graph-1ab93-5c487f9bf4-zvtmk namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 7774d6c0-f8dc-4d18-8782-5d3488aec90f kind: Event lastTimestamp: "2026-07-16T22:12:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7e35538adc84c05d9420d87001e9355aeddffd6326bb391010f8e6c1a3c85203" already present on machine metadata: creationTimestamp: "2026-07-16T22:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:26Z" name: ensemble-graph-1ab93-5c487f9bf4-zvtmk.18c2e4c7389e7e81 namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 3bd5e9b1-5f46-4045-b409-ff706725f1d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1ab93} kind: Pod name: ensemble-graph-1ab93-5c487f9bf4-zvtmk namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 7774d6c0-f8dc-4d18-8782-5d3488aec90f kind: Event lastTimestamp: "2026-07-16T22:12:26Z" message: 'Created container: ensemble-graph-1ab93' metadata: creationTimestamp: "2026-07-16T22:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:26Z" name: ensemble-graph-1ab93-5c487f9bf4-zvtmk.18c2e4c73bb104d0 namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: 55dc2d96-7c16-4497-8b88-37033f55eeaf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1ab93} kind: Pod name: ensemble-graph-1ab93-5c487f9bf4-zvtmk namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 7774d6c0-f8dc-4d18-8782-5d3488aec90f kind: Event lastTimestamp: "2026-07-16T22:12:26Z" message: Started container ensemble-graph-1ab93 metadata: creationTimestamp: "2026-07-16T22:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:26Z" name: ensemble-graph-1ab93-5c487f9bf4-zvtmk.18c2e4c73c283787 namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: b7bda1ca-eced-4cc5-94b0-788324a02bce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1ab93} kind: Pod name: ensemble-graph-1ab93-5c487f9bf4-zvtmk namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 7774d6c0-f8dc-4d18-8782-5d3488aec90f kind: Event lastTimestamp: "2026-07-16T22:12:36Z" message: Stopping container ensemble-graph-1ab93 metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:36Z" name: ensemble-graph-1ab93-5c487f9bf4-zvtmk.18c2e4c97620653d namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 660d279e-c53f-4b95-aa09-2ee518aa2a80 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:12:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1ab93} kind: Pod name: ensemble-graph-1ab93-5c487f9bf4-zvtmk namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: 7774d6c0-f8dc-4d18-8782-5d3488aec90f kind: Event lastTimestamp: "2026-07-16T22:13:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T22:12:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:03Z" name: ensemble-graph-1ab93-5c487f9bf4-zvtmk.18c2e4c9f1d44778 namespace: kserve-ci-e2e-test resourceVersion: "21546" uid: 07aa0cb9-4d5f-45c8-a6c3-c5ea1416c3a7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-1ab93-5c487f9bf4 namespace: kserve-ci-e2e-test resourceVersion: "21028" uid: a5f1d9c3-9ce3-478d-a1fd-6e82b3987eee kind: Event lastTimestamp: "2026-07-16T22:12:26Z" message: 'Created pod: ensemble-graph-1ab93-5c487f9bf4-zvtmk' metadata: creationTimestamp: "2026-07-16T22:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:26Z" name: ensemble-graph-1ab93-5c487f9bf4.18c2e4c71d343122 namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: d5df65f1-e4c4-45ae-9361-c3146c7e236e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-1ab93 namespace: kserve-ci-e2e-test resourceVersion: "21026" uid: b1ffd482-23c0-4a48-978d-5dbb5e0d76f0 kind: Event lastTimestamp: "2026-07-16T22:12:26Z" message: Scaled up replica set ensemble-graph-1ab93-5c487f9bf4 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:26Z" name: ensemble-graph-1ab93.18c2e4c71c34b201 namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 1961eba9-2977-433b-b72f-9b06d1f6e7ef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-1ab93 namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: 0286ba68-fec8-4510-ac75-fb0758efeead kind: Event lastTimestamp: "2026-07-16T22:12:26Z" message: InferenceGraph [ensemble-graph-1ab93] is Ready metadata: creationTimestamp: "2026-07-16T22:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:12:26Z" name: ensemble-graph-1ab93.18c2e4c7205e822d namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: 2943277d-7571-4d5f-8b9f-c3a3d9b5466b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:13:46.613830Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c1850-7f4cf9486c-vknjg namespace: kserve-ci-e2e-test resourceVersion: "22089" uid: c0d8e27b-0eba-4c02-b3b0-f50f5ddcedc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-c1850-7f4cf9486c-vknjg to ip-10-0-131-231.ec2.internal metadata: creationTimestamp: "2026-07-16T22:13:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:13:46Z" name: ensemble-graph-c1850-7f4cf9486c-vknjg.18c2e4d9d459f936 namespace: kserve-ci-e2e-test resourceVersion: "22095" uid: 08b17cfa-7e41-4653-a879-c8a4d9e35b59 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:47Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c1850-7f4cf9486c-vknjg namespace: kserve-ci-e2e-test resourceVersion: "22098" uid: c0d8e27b-0eba-4c02-b3b0-f50f5ddcedc7 kind: Event lastTimestamp: "2026-07-16T22:13:47Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:13:47Z" name: ensemble-graph-c1850-7f4cf9486c-vknjg.18c2e4d9ede2f71f namespace: kserve-ci-e2e-test resourceVersion: "22110" uid: 4a39aac6-1310-4ff6-8c1b-0b251bb848ea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c1850} kind: Pod name: ensemble-graph-c1850-7f4cf9486c-vknjg namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: c0d8e27b-0eba-4c02-b3b0-f50f5ddcedc7 kind: Event lastTimestamp: "2026-07-16T22:13:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7e35538adc84c05d9420d87001e9355aeddffd6326bb391010f8e6c1a3c85203" already present on machine metadata: creationTimestamp: "2026-07-16T22:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:47Z" name: ensemble-graph-c1850-7f4cf9486c-vknjg.18c2e4d9f08703f9 namespace: kserve-ci-e2e-test resourceVersion: "22113" uid: 9ab4ef00-eee3-412c-9a86-339783d80455 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c1850} kind: Pod name: ensemble-graph-c1850-7f4cf9486c-vknjg namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: c0d8e27b-0eba-4c02-b3b0-f50f5ddcedc7 kind: Event lastTimestamp: "2026-07-16T22:13:47Z" message: 'Created container: ensemble-graph-c1850' metadata: creationTimestamp: "2026-07-16T22:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:47Z" name: ensemble-graph-c1850-7f4cf9486c-vknjg.18c2e4d9f37e8636 namespace: kserve-ci-e2e-test resourceVersion: "22114" uid: 4719ee3c-b068-4d71-be12-c14809d3bcd3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c1850} kind: Pod name: ensemble-graph-c1850-7f4cf9486c-vknjg namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: c0d8e27b-0eba-4c02-b3b0-f50f5ddcedc7 kind: Event lastTimestamp: "2026-07-16T22:13:47Z" message: Started container ensemble-graph-c1850 metadata: creationTimestamp: "2026-07-16T22:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:47Z" name: ensemble-graph-c1850-7f4cf9486c-vknjg.18c2e4d9f3f2ebe1 namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: a4cbeaea-6f8d-425f-8f24-f62a4b210ba6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c1850} kind: Pod name: ensemble-graph-c1850-7f4cf9486c-vknjg namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: c0d8e27b-0eba-4c02-b3b0-f50f5ddcedc7 kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: Stopping container ensemble-graph-c1850 metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:01Z" name: ensemble-graph-c1850-7f4cf9486c-vknjg.18c2e54cfc8088fa namespace: kserve-ci-e2e-test resourceVersion: "25877" uid: e7dcb762-8a26-4aed-a1ff-5c6dbd790cfc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:22:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c1850} kind: Pod name: ensemble-graph-c1850-7f4cf9486c-vknjg namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: c0d8e27b-0eba-4c02-b3b0-f50f5ddcedc7 kind: Event lastTimestamp: "2026-07-16T22:22:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T22:22:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:28Z" name: ensemble-graph-c1850-7f4cf9486c-vknjg.18c2e54d90174f72 namespace: kserve-ci-e2e-test resourceVersion: "26215" uid: b4874193-3785-481f-b38e-b2e98f8d48e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-c1850-7f4cf9486c namespace: kserve-ci-e2e-test resourceVersion: "22085" uid: cd9065ae-b8be-436f-ba32-b859a679e94a kind: Event lastTimestamp: "2026-07-16T22:13:46Z" message: 'Created pod: ensemble-graph-c1850-7f4cf9486c-vknjg' metadata: creationTimestamp: "2026-07-16T22:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:46Z" name: ensemble-graph-c1850-7f4cf9486c.18c2e4d9d3c7db54 namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: 495ee957-f890-4ee1-9032-b5f330519e45 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-c1850 namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: 9f1d89c0-8838-4901-a687-eeef042387e9 kind: Event lastTimestamp: "2026-07-16T22:13:46Z" message: Scaled up replica set ensemble-graph-c1850-7f4cf9486c from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:46Z" name: ensemble-graph-c1850.18c2e4d9d25227f5 namespace: kserve-ci-e2e-test resourceVersion: "22086" uid: 730b2f00-15c6-486a-b988-7dc854b42a62 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c1850 namespace: kserve-ci-e2e-test resourceVersion: "22101" uid: 40b0d9d6-14ce-455b-8280-5d0b34175f77 kind: Event lastTimestamp: "2026-07-16T22:13:46Z" message: InferenceGraph [ensemble-graph-c1850] is Ready metadata: creationTimestamp: "2026-07-16T22:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:13:46Z" name: ensemble-graph-c1850.18c2e4d9d9d66664 namespace: kserve-ci-e2e-test resourceVersion: "22103" uid: 84e51890-adfd-4067-8e41-863da13e93f0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c1850 namespace: kserve-ci-e2e-test resourceVersion: "22084" uid: da7e1c21-8d33-4a87-8319-77da66c446f2 kind: Event lastTimestamp: "2026-07-16T22:14:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:01Z" name: ensemble-graph-c1850.18c2e4dd51065bc3 namespace: kserve-ci-e2e-test resourceVersion: "22228" uid: 0f095e85-1354-498e-8d24-678f5e4eb8c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c1850 namespace: kserve-ci-e2e-test resourceVersion: "22084" uid: da7e1c21-8d33-4a87-8319-77da66c446f2 kind: Event lastTimestamp: "2026-07-16T22:14:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:01Z" name: ensemble-graph-c1850.18c2e4dd51848d67 namespace: kserve-ci-e2e-test resourceVersion: "22229" uid: 39904578-b8dd-4c60-95ae-569a71149b13 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c1850 namespace: kserve-ci-e2e-test resourceVersion: "22227" uid: da7e1c21-8d33-4a87-8319-77da66c446f2 kind: Event lastTimestamp: "2026-07-16T22:14:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:14:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:16Z" name: ensemble-graph-c1850.18c2e4e0d0407425 namespace: kserve-ci-e2e-test resourceVersion: "22358" uid: cd3a6c6c-1c57-4c24-a728-07da83b861eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c1850 namespace: kserve-ci-e2e-test resourceVersion: "22227" uid: da7e1c21-8d33-4a87-8319-77da66c446f2 kind: Event lastTimestamp: "2026-07-16T22:14:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:14:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:16Z" name: ensemble-graph-c1850.18c2e4e0d0af4e69 namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: df1f567b-8dad-4c9a-9e95-cf561215b183 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:11:16.307817Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: 6ce564fd-bfcf-4e1b-8430-7347fb64adaa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1ab93-predictor-7cd97fc847-qltmd to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd.18c2e4b6d56a443a namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: 313eed06-40e2-4023-9fa2-65295c944c08 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: 6ce564fd-bfcf-4e1b-8430-7347fb64adaa kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd.18c2e4b6f010c7b8 namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: 70f26af4-44c8-48a3-b683-6648a1dff1e9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 6ce564fd-bfcf-4e1b-8430-7347fb64adaa kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd.18c2e4b6f18afdbb namespace: kserve-ci-e2e-test resourceVersion: "20280" uid: 7e039479-807c-4226-b266-a1f75e09b717 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 6ce564fd-bfcf-4e1b-8430-7347fb64adaa kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd.18c2e4b6f5440427 namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: 37de934e-b14a-4cdc-b89c-84bf3d5da471 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 6ce564fd-bfcf-4e1b-8430-7347fb64adaa kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd.18c2e4b6f5dca3b7 namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: d1c8b890-e5d6-4a32-a8f0-99345e41d7b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 6ce564fd-bfcf-4e1b-8430-7347fb64adaa kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd.18c2e4b6f5e76951 namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: 0a69cb56-7992-49ea-96a3-03610df62660 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 6ce564fd-bfcf-4e1b-8430-7347fb64adaa kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd.18c2e4b6f9f456a7 namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: 55691513-527f-499b-8767-8d98d7c343c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 6ce564fd-bfcf-4e1b-8430-7347fb64adaa kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd.18c2e4b6fa76876f namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: 18d95a7c-5cd2-4cc6-9a72-768d323317b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T22:11:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 6ce564fd-bfcf-4e1b-8430-7347fb64adaa kind: Event lastTimestamp: "2026-07-16T22:12:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:11:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:04Z" name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd.18c2e4b757724eaa namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: ea35aa8f-fcdb-4edc-9b2a-6a81eac2e3f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 6ce564fd-bfcf-4e1b-8430-7347fb64adaa kind: Event lastTimestamp: "2026-07-16T22:12:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:36Z" name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd.18c2e4c9837b6f8e namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: ccb007b8-34d1-491a-9acc-b0408300df08 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 6ce564fd-bfcf-4e1b-8430-7347fb64adaa kind: Event lastTimestamp: "2026-07-16T22:12:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:36Z" name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd.18c2e4c9837c79eb namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: 48ccdfbc-2f34-455d-9907-f4757aaf1e5d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: 6ce564fd-bfcf-4e1b-8430-7347fb64adaa kind: Event lastTimestamp: "2026-07-16T22:12:39Z" message: 'Readiness probe failed: Get "https://10.134.0.46:8643/healthz": dial tcp 10.134.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:12:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:39Z" name: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd.18c2e4ca33a654d4 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 561331ae-1288-48b6-8884-fab99b494f39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1ab93-predictor-7cd97fc847 namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 1185ab1a-2035-4f38-a84e-3aa166f0e08b kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: 'Created pod: error-404-isvc-1ab93-predictor-7cd97fc847-qltmd' metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-1ab93-predictor-7cd97fc847.18c2e4b6d4d573fc namespace: kserve-ci-e2e-test resourceVersion: "20256" uid: 43ef77dc-e6ba-4353-b6b3-3e00b2cdbbb9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1ab93-predictor namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: 7eb2bb12-f805-413d-8320-4a5160143d84 kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: Scaled up replica set error-404-isvc-1ab93-predictor-7cd97fc847 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-1ab93-predictor.18c2e4b6d274a062 namespace: kserve-ci-e2e-test resourceVersion: "20244" uid: 6ee7d289-d9ef-4b85-b7d0-8177a1aafbfe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ab93-predictor namespace: kserve-ci-e2e-test resourceVersion: "20240" uid: 07ec852f-6f46-4b88-be33-28bd3184d38b kind: Event lastTimestamp: "2026-07-16T22:11:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:11:31Z" name: error-404-isvc-1ab93-predictor.18c2e4ba5107642c namespace: kserve-ci-e2e-test resourceVersion: "20421" uid: 0c1e0be1-9059-4175-99ae-edafefe08ef8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ab93-predictor namespace: kserve-ci-e2e-test resourceVersion: "20240" uid: 07ec852f-6f46-4b88-be33-28bd3184d38b kind: Event lastTimestamp: "2026-07-16T22:11:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:11:31Z" name: error-404-isvc-1ab93-predictor.18c2e4ba51798b77 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 562f042f-bf01-4464-bcab-7c9ae6b2eba3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:11:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ab93-predictor namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: 07ec852f-6f46-4b88-be33-28bd3184d38b kind: Event lastTimestamp: "2026-07-16T22:12:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:31Z" name: error-404-isvc-1ab93-predictor.18c2e4bdd050ef1a namespace: kserve-ci-e2e-test resourceVersion: "21094" uid: 17f1d24d-831d-4267-97c5-e5d61ad28b78 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:11:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ab93-predictor namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: 07ec852f-6f46-4b88-be33-28bd3184d38b kind: Event lastTimestamp: "2026-07-16T22:12:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:31Z" name: error-404-isvc-1ab93-predictor.18c2e4bdd0c9aff1 namespace: kserve-ci-e2e-test resourceVersion: "21095" uid: 753c6570-1e9b-4e00-a24b-72922fa3c67b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ab93 namespace: kserve-ci-e2e-test resourceVersion: "20212" uid: 77d03bf4-2c82-4512-8c55-b7cd55561ad9 kind: Event lastTimestamp: "2026-07-16T22:12:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:12:14Z" name: error-404-isvc-1ab93.18c2e4b6d27e9880 namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: a9cae51e-aa4c-4763-91c4-9f8b86587c33 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ab93 namespace: kserve-ci-e2e-test resourceVersion: "20212" uid: 77d03bf4-2c82-4512-8c55-b7cd55561ad9 kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: 'Failed to update status for InferenceService "error-404-isvc-1ab93": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1ab93": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-1ab93.18c2e4b6d34680a0 namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: 3de26155-2a01-4ce4-b267-18d256629607 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ab93 namespace: kserve-ci-e2e-test resourceVersion: "20212" uid: 77d03bf4-2c82-4512-8c55-b7cd55561ad9 kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1ab93": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-1ab93.18c2e4b6d346dd7a namespace: kserve-ci-e2e-test resourceVersion: "20251" uid: 295f0226-4ef3-4e54-8807-74499f9be255 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ab93 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 77d03bf4-2c82-4512-8c55-b7cd55561ad9 kind: Event lastTimestamp: "2026-07-16T22:12:14Z" message: InferenceService [error-404-isvc-1ab93] is Ready metadata: creationTimestamp: "2026-07-16T22:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:12:14Z" name: error-404-isvc-1ab93.18c2e4c46743989d namespace: kserve-ci-e2e-test resourceVersion: "20918" uid: e8298bf4-c896-45e2-880b-e0cf21706d65 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:11:52.224800Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 25fc5316-4d1e-480a-a6e4-db45339a8d09 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-67446-predictor-86c85bf98f-wwgzs to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:11:52Z" name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs.18c2e4bf323be383 namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: dd046583-20b4-41a3-bbb8-b2391eeca543 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: 25fc5316-4d1e-480a-a6e4-db45339a8d09 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:11:52Z" name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs.18c2e4bf4cd8b570 namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 7def3edb-74e7-4c93-a0fc-2b09fd1c7795 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 25fc5316-4d1e-480a-a6e4-db45339a8d09 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:52Z" name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs.18c2e4bf4e542396 namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: 2a2af43a-5f4b-4218-bedf-5beb76a13d35 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 25fc5316-4d1e-480a-a6e4-db45339a8d09 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:52Z" name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs.18c2e4bf51b052e2 namespace: kserve-ci-e2e-test resourceVersion: "20701" uid: d35dd645-f4e1-4209-8471-497514b03570 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 25fc5316-4d1e-480a-a6e4-db45339a8d09 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:52Z" name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs.18c2e4bf522bb41f namespace: kserve-ci-e2e-test resourceVersion: "20702" uid: 3708a543-0bf5-41fb-a698-bb0ce4f0984d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 25fc5316-4d1e-480a-a6e4-db45339a8d09 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:52Z" name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs.18c2e4bf523879b6 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 02cc2514-f5f5-4cd1-8d2f-e91a06e5a6e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 25fc5316-4d1e-480a-a6e4-db45339a8d09 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:52Z" name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs.18c2e4bf560e6364 namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 7a0b1bce-4e0a-4716-a879-2d232bee3202 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 25fc5316-4d1e-480a-a6e4-db45339a8d09 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:52Z" name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs.18c2e4bf568de515 namespace: kserve-ci-e2e-test resourceVersion: "20705" uid: d8df9b48-d9dc-48df-86d8-2f88d3ffaba2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T22:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 25fc5316-4d1e-480a-a6e4-db45339a8d09 kind: Event lastTimestamp: "2026-07-16T22:12:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:40Z" name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs.18c2e4bfc5a4c822 namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: d2f21d25-78ba-464f-8786-01f941d764a6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 25fc5316-4d1e-480a-a6e4-db45339a8d09 kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:12Z" name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs.18c2e4d1d9ccd68b namespace: kserve-ci-e2e-test resourceVersion: "21656" uid: b38b4907-9ba0-478e-a88d-0438e657cd0d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 25fc5316-4d1e-480a-a6e4-db45339a8d09 kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:12Z" name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs.18c2e4d1d9cdfe84 namespace: kserve-ci-e2e-test resourceVersion: "21658" uid: d6c61b50-1160-488d-9f58-e567b33b0cb6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 25fc5316-4d1e-480a-a6e4-db45339a8d09 kind: Event lastTimestamp: "2026-07-16T22:13:15Z" message: 'Readiness probe failed: Get "https://10.134.0.48:8643/healthz": dial tcp 10.134.0.48:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:15Z" name: error-404-isvc-67446-predictor-86c85bf98f-wwgzs.18c2e4d2a1e5bce3 namespace: kserve-ci-e2e-test resourceVersion: "21795" uid: 6d35e735-efde-4c9f-bd29-2b25f06e44a7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-67446-predictor-86c85bf98f namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 78fdc433-08dc-45bd-b656-e0ae8f5bd0f6 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: 'Created pod: error-404-isvc-67446-predictor-86c85bf98f-wwgzs' metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:11:52Z" name: error-404-isvc-67446-predictor-86c85bf98f.18c2e4bf31a4869a namespace: kserve-ci-e2e-test resourceVersion: "20675" uid: f93f2899-64f4-4da4-8758-dec83dae9df1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-67446-predictor namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 4cd2ce3f-bc14-43b3-b8ea-401e78fbfa4b kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: Scaled up replica set error-404-isvc-67446-predictor-86c85bf98f from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:11:52Z" name: error-404-isvc-67446-predictor.18c2e4bf2fcbc6b7 namespace: kserve-ci-e2e-test resourceVersion: "20666" uid: 67f82439-2a21-4c11-9ef5-ad9fd5a435b6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-67446-predictor namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 962eec13-b4f1-4379-a102-b845d25f8f75 kind: Event lastTimestamp: "2026-07-16T22:12:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:07Z" name: error-404-isvc-67446-predictor.18c2e4c2ae657b47 namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: 01347e30-0098-4311-9b83-ac3275f37b30 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-67446-predictor namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 962eec13-b4f1-4379-a102-b845d25f8f75 kind: Event lastTimestamp: "2026-07-16T22:12:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:07Z" name: error-404-isvc-67446-predictor.18c2e4c2aeb0daec namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 88fa0db3-c6d4-4605-b01a-f71514ea1b8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:12:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-67446-predictor namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 962eec13-b4f1-4379-a102-b845d25f8f75 kind: Event lastTimestamp: "2026-07-16T22:13:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:07Z" name: error-404-isvc-67446-predictor.18c2e4c62d744640 namespace: kserve-ci-e2e-test resourceVersion: "21585" uid: eda98569-96bf-4a5e-87cd-a50b9aa37941 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:12:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-67446-predictor namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 962eec13-b4f1-4379-a102-b845d25f8f75 kind: Event lastTimestamp: "2026-07-16T22:13:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:07Z" name: error-404-isvc-67446-predictor.18c2e4c62dfa6829 namespace: kserve-ci-e2e-test resourceVersion: "21586" uid: 7246b716-2904-4082-921e-13b69dc3f849 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-67446 namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: 8daae53f-8ebe-4bbd-8937-d8a140434b5a kind: Event lastTimestamp: "2026-07-16T22:12:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:12:50Z" name: error-404-isvc-67446.18c2e4bf2fce1715 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 1e7fffbb-f6f0-4b27-8293-984513945c75 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-67446 namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: 8daae53f-8ebe-4bbd-8937-d8a140434b5a kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: 'Failed to update status for InferenceService "error-404-isvc-67446": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-67446": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:11:52Z" name: error-404-isvc-67446.18c2e4bf306b3d71 namespace: kserve-ci-e2e-test resourceVersion: "20669" uid: 1c3e5f05-d01c-45cf-b9ad-6d7aac632e62 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-67446 namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: 8daae53f-8ebe-4bbd-8937-d8a140434b5a kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-67446": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:11:52Z" name: error-404-isvc-67446.18c2e4bf306b8d34 namespace: kserve-ci-e2e-test resourceVersion: "20670" uid: f9565958-c72f-4bf6-85d6-305e33c10670 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-67446 namespace: kserve-ci-e2e-test resourceVersion: "21410" uid: 8daae53f-8ebe-4bbd-8937-d8a140434b5a kind: Event lastTimestamp: "2026-07-16T22:12:50Z" message: InferenceService [error-404-isvc-67446] is Ready metadata: creationTimestamp: "2026-07-16T22:12:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:12:50Z" name: error-404-isvc-67446.18c2e4ccda561422 namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: 7d69bfd2-2fda-4446-97ee-75ff803fb044 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:01:51.485925Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-69517-predictor-86957765c5-22c7r namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: 36437163-7ef1-4122-93fc-808ae5ae10a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-69517-predictor-86957765c5-22c7r to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:01:51Z" name: error-404-isvc-69517-predictor-86957765c5-22c7r.18c2e43353682341 namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: f16e3392-7ac6-413a-9dcb-2d1db0548734 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-69517-predictor-86957765c5-22c7r namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 36437163-7ef1-4122-93fc-808ae5ae10a1 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-69517-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:51Z" name: error-404-isvc-69517-predictor-86957765c5-22c7r.18c2e43362049210 namespace: kserve-ci-e2e-test resourceVersion: "15800" uid: 614f0e4f-6700-4aa0-8044-aeaec7600c37 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-69517-predictor-86957765c5-22c7r namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: 36437163-7ef1-4122-93fc-808ae5ae10a1 kind: Event lastTimestamp: "2026-07-16T22:01:52Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:01:52Z" name: error-404-isvc-69517-predictor-86957765c5-22c7r.18c2e43391a2c0bd namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: 57e9d82a-3dd0-4a81-8bfb-13b7b54a1821 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-69517-predictor-86957765c5-22c7r namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 36437163-7ef1-4122-93fc-808ae5ae10a1 kind: Event lastTimestamp: "2026-07-16T22:01:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:52Z" name: error-404-isvc-69517-predictor-86957765c5-22c7r.18c2e433931b0c72 namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: 4a883e58-5953-4814-8517-bb6a53253118 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-69517-predictor-86957765c5-22c7r namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 36437163-7ef1-4122-93fc-808ae5ae10a1 kind: Event lastTimestamp: "2026-07-16T22:01:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:52Z" name: error-404-isvc-69517-predictor-86957765c5-22c7r.18c2e43396f901f7 namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: 443efcfe-d435-46f3-945d-f3d2076a0468 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-69517-predictor-86957765c5-22c7r namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 36437163-7ef1-4122-93fc-808ae5ae10a1 kind: Event lastTimestamp: "2026-07-16T22:01:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:52Z" name: error-404-isvc-69517-predictor-86957765c5-22c7r.18c2e433977534bc namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: 09bd45f1-96bf-421a-8eeb-70652e1ba580 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-69517-predictor-86957765c5-22c7r namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 36437163-7ef1-4122-93fc-808ae5ae10a1 kind: Event lastTimestamp: "2026-07-16T22:01:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:52Z" name: error-404-isvc-69517-predictor-86957765c5-22c7r.18c2e433978197f7 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 891cdfd9-7ad5-4a0a-9890-b1b45665d2a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-69517-predictor-86957765c5-22c7r namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 36437163-7ef1-4122-93fc-808ae5ae10a1 kind: Event lastTimestamp: "2026-07-16T22:01:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:52Z" name: error-404-isvc-69517-predictor-86957765c5-22c7r.18c2e4339aa8b9e7 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 5a73f22d-1a1c-4f07-bd7d-e11af0aee022 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-69517-predictor-86957765c5-22c7r namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 36437163-7ef1-4122-93fc-808ae5ae10a1 kind: Event lastTimestamp: "2026-07-16T22:01:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:52Z" name: error-404-isvc-69517-predictor-86957765c5-22c7r.18c2e4339b26e16c namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 7db0d4b3-9cbb-4fe3-a299-b0f23f3bebfd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:01:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-69517-predictor-86957765c5-22c7r namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 36437163-7ef1-4122-93fc-808ae5ae10a1 kind: Event lastTimestamp: "2026-07-16T22:02:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:30Z" name: error-404-isvc-69517-predictor-86957765c5-22c7r.18c2e4340af8f8d5 namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: aa9b2e00-36f4-42c0-b4a8-feeec260e7b3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-69517-predictor-86957765c5-22c7r namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 36437163-7ef1-4122-93fc-808ae5ae10a1 kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-69517-predictor-86957765c5-22c7r.18c2e4b6d23a18ce namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: 2a0c8971-685d-4163-8bbd-78aae2f0a538 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-69517-predictor-86957765c5-22c7r namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 36437163-7ef1-4122-93fc-808ae5ae10a1 kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:16Z" name: error-404-isvc-69517-predictor-86957765c5-22c7r.18c2e4b6d23b1832 namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: f2d10fdb-064e-4cc8-bc32-d25763ae5421 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-69517-predictor-86957765c5 namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: 6bf125e8-85c5-419b-805e-1ac80aab33f9 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: 'Created pod: error-404-isvc-69517-predictor-86957765c5-22c7r' metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:51Z" name: error-404-isvc-69517-predictor-86957765c5.18c2e43352f6a9c5 namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: 2a7becd2-2cab-4533-990d-3eb447905fca reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-69517-predictor namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 57e86ae2-8e7e-4926-b78e-7224e9cba319 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: Scaled up replica set error-404-isvc-69517-predictor-86957765c5 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:51Z" name: error-404-isvc-69517-predictor.18c2e433511c653b namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: dad9b573-71fb-427a-9dec-191bc3f5f8af reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:02:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-69517-predictor namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: dca3851b-b7c6-433b-ab95-dd2617f2c1c0 kind: Event lastTimestamp: "2026-07-16T22:02:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:21Z" name: error-404-isvc-69517-predictor.18c2e436cfcbd0b5 namespace: kserve-ci-e2e-test resourceVersion: "16099" uid: 1a451e04-d1a4-4ce5-8fbd-3dc9df3910c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:02:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-69517-predictor namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: dca3851b-b7c6-433b-ab95-dd2617f2c1c0 kind: Event lastTimestamp: "2026-07-16T22:02:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:21Z" name: error-404-isvc-69517-predictor.18c2e436d03344f5 namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: b3ac2ad5-abed-45cf-94b6-65f7ba98bef0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:02:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-69517-predictor namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: dca3851b-b7c6-433b-ab95-dd2617f2c1c0 kind: Event lastTimestamp: "2026-07-16T22:03:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:06Z" name: error-404-isvc-69517-predictor.18c2e43dcdf7f5df namespace: kserve-ci-e2e-test resourceVersion: "16724" uid: bd2f6c57-cad9-4988-8b26-bc98cbaa7915 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:02:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-69517-predictor namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: dca3851b-b7c6-433b-ab95-dd2617f2c1c0 kind: Event lastTimestamp: "2026-07-16T22:03:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:06Z" name: error-404-isvc-69517-predictor.18c2e43dce6fb60c namespace: kserve-ci-e2e-test resourceVersion: "16725" uid: fd40978d-4cb1-4d73-a921-34433d2604ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-69517 namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: 608791f2-b930-4736-a4ad-1cd9322d7a44 kind: Event lastTimestamp: "2026-07-16T22:02:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:02:40Z" name: error-404-isvc-69517.18c2e433511b7cdc namespace: kserve-ci-e2e-test resourceVersion: "16464" uid: 74aba95c-8464-479c-98bf-22bc090b86c1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-69517 namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: 608791f2-b930-4736-a4ad-1cd9322d7a44 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-69517": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-69517": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:01:51Z" name: error-404-isvc-69517.18c2e43351daa35e namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: 1b036c0c-cfd2-4e3f-aa3f-564d63ddd97f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-69517 namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: 608791f2-b930-4736-a4ad-1cd9322d7a44 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-69517": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:01:51Z" name: error-404-isvc-69517.18c2e43351dae2fd namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: 0252ccf6-db07-4827-a93d-3af5921eaa5c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-69517 namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: 608791f2-b930-4736-a4ad-1cd9322d7a44 kind: Event lastTimestamp: "2026-07-16T22:02:40Z" message: InferenceService [error-404-isvc-69517] is Ready metadata: creationTimestamp: "2026-07-16T22:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:02:40Z" name: error-404-isvc-69517.18c2e43ec6bed32b namespace: kserve-ci-e2e-test resourceVersion: "16462" uid: 2f6617ea-4db0-48dd-ac86-fb97f27cd675 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:23:22.026569Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-76a97-predictor-f65b59759-wj9k2 namespace: kserve-ci-e2e-test resourceVersion: "26953" uid: 3c1c56da-d890-4369-be55-ce9b8cfee7b9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-76a97-predictor-f65b59759-wj9k2 to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:23:22Z" name: error-404-isvc-76a97-predictor-f65b59759-wj9k2.18c2e55fcd9f9738 namespace: kserve-ci-e2e-test resourceVersion: "26957" uid: 19b30a25-6fcd-478d-b3f4-e1546eceb0f6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-76a97-predictor-f65b59759-wj9k2 namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: 3c1c56da-d890-4369-be55-ce9b8cfee7b9 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: Add eth0 [10.134.0.58/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:23:22Z" name: error-404-isvc-76a97-predictor-f65b59759-wj9k2.18c2e55fe8eac394 namespace: kserve-ci-e2e-test resourceVersion: "26985" uid: ea3c08ec-38d2-4d71-8840-434aba3d8754 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-76a97-predictor-f65b59759-wj9k2 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 3c1c56da-d890-4369-be55-ce9b8cfee7b9 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:22Z" name: error-404-isvc-76a97-predictor-f65b59759-wj9k2.18c2e55feac2a2b1 namespace: kserve-ci-e2e-test resourceVersion: "26987" uid: 22218bfc-db68-46f5-a600-667b55cc4f67 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-76a97-predictor-f65b59759-wj9k2 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 3c1c56da-d890-4369-be55-ce9b8cfee7b9 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:22Z" name: error-404-isvc-76a97-predictor-f65b59759-wj9k2.18c2e55feedb825e namespace: kserve-ci-e2e-test resourceVersion: "26991" uid: c0660b5a-c530-4b3f-9970-a434a0aeae5c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-76a97-predictor-f65b59759-wj9k2 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 3c1c56da-d890-4369-be55-ce9b8cfee7b9 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:22Z" name: error-404-isvc-76a97-predictor-f65b59759-wj9k2.18c2e55fef551b82 namespace: kserve-ci-e2e-test resourceVersion: "26993" uid: 6597f2d3-198e-4437-9bee-f9e22828ccad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-76a97-predictor-f65b59759-wj9k2 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 3c1c56da-d890-4369-be55-ce9b8cfee7b9 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:22Z" name: error-404-isvc-76a97-predictor-f65b59759-wj9k2.18c2e55fef60b683 namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: d1c7a541-7a1d-4e7d-920b-885969b2798f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-76a97-predictor-f65b59759-wj9k2 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 3c1c56da-d890-4369-be55-ce9b8cfee7b9 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:22Z" name: error-404-isvc-76a97-predictor-f65b59759-wj9k2.18c2e55ff2ff4991 namespace: kserve-ci-e2e-test resourceVersion: "27000" uid: 2e63cd7e-85a0-4b42-96c6-59ed74b49419 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-76a97-predictor-f65b59759-wj9k2 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 3c1c56da-d890-4369-be55-ce9b8cfee7b9 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:22Z" name: error-404-isvc-76a97-predictor-f65b59759-wj9k2.18c2e55ff37a1c8d namespace: kserve-ci-e2e-test resourceVersion: "27001" uid: 9cf9f24f-2865-4233-9ce8-ca785e720501 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-16T22:23:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-76a97-predictor-f65b59759-wj9k2 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 3c1c56da-d890-4369-be55-ce9b8cfee7b9 kind: Event lastTimestamp: "2026-07-16T22:32:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.58:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:23:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:32:39Z" name: error-404-isvc-76a97-predictor-f65b59759-wj9k2.18c2e560227a1c2b namespace: kserve-ci-e2e-test resourceVersion: "31072" uid: 355255ed-62ae-4e25-8eb4-a95467309553 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-76a97-predictor-f65b59759-wj9k2 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 3c1c56da-d890-4369-be55-ce9b8cfee7b9 kind: Event lastTimestamp: "2026-07-16T22:32:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:32:36Z" name: error-404-isvc-76a97-predictor-f65b59759-wj9k2.18c2e5e0feda8635 namespace: kserve-ci-e2e-test resourceVersion: "31050" uid: 3fd205f2-b354-4744-abc7-8ae1aeb64df4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-76a97-predictor-f65b59759-wj9k2 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 3c1c56da-d890-4369-be55-ce9b8cfee7b9 kind: Event lastTimestamp: "2026-07-16T22:32:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:32:36Z" name: error-404-isvc-76a97-predictor-f65b59759-wj9k2.18c2e5e0fedafaee namespace: kserve-ci-e2e-test resourceVersion: "31051" uid: d7aaf39a-06b3-473b-803b-b5763c82a830 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:32:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-76a97-predictor-f65b59759-wj9k2 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 3c1c56da-d890-4369-be55-ce9b8cfee7b9 kind: Event lastTimestamp: "2026-07-16T22:32:39Z" message: 'Readiness probe failed: Get "https://10.134.0.58:8643/healthz": dial tcp 10.134.0.58:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:32:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:32:39Z" name: error-404-isvc-76a97-predictor-f65b59759-wj9k2.18c2e5e19714acf1 namespace: kserve-ci-e2e-test resourceVersion: "31071" uid: dfc8ed73-bdf1-4d4d-9c63-4dd931a899f1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-76a97-predictor-f65b59759 namespace: kserve-ci-e2e-test resourceVersion: "26943" uid: e8db7917-27a6-4aa6-8fb2-98652b6dc415 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: 'Created pod: error-404-isvc-76a97-predictor-f65b59759-wj9k2' metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:22Z" name: error-404-isvc-76a97-predictor-f65b59759.18c2e55fccd137b1 namespace: kserve-ci-e2e-test resourceVersion: "26956" uid: c3588f3a-a75e-478a-85bb-cebd31122535 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-76a97-predictor namespace: kserve-ci-e2e-test resourceVersion: "26940" uid: d77eb166-c42e-4233-9ffe-5c4d192316de kind: Event lastTimestamp: "2026-07-16T22:23:21Z" message: Scaled up replica set error-404-isvc-76a97-predictor-f65b59759 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:21Z" name: error-404-isvc-76a97-predictor.18c2e55fca9bdc1a namespace: kserve-ci-e2e-test resourceVersion: "26945" uid: d189aa3e-1e8c-4421-bd9b-e38d646706e2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-76a97-predictor namespace: kserve-ci-e2e-test resourceVersion: "26942" uid: beccadd8-e8df-49b5-9cdd-0dcd782dd527 kind: Event lastTimestamp: "2026-07-16T22:23:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:36Z" name: error-404-isvc-76a97-predictor.18c2e563493afcdd namespace: kserve-ci-e2e-test resourceVersion: "27143" uid: a81b4f2d-afe3-4363-85ae-7ea2e5b2053e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-76a97-predictor namespace: kserve-ci-e2e-test resourceVersion: "26942" uid: beccadd8-e8df-49b5-9cdd-0dcd782dd527 kind: Event lastTimestamp: "2026-07-16T22:23:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:36Z" name: error-404-isvc-76a97-predictor.18c2e5634a00d18e namespace: kserve-ci-e2e-test resourceVersion: "27145" uid: 9aded3ca-e9e9-4922-8785-eb089b4fdace reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:23:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-76a97-predictor namespace: kserve-ci-e2e-test resourceVersion: "27144" uid: beccadd8-e8df-49b5-9cdd-0dcd782dd527 kind: Event lastTimestamp: "2026-07-16T22:24:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:52Z" name: error-404-isvc-76a97-predictor.18c2e566c90f1993 namespace: kserve-ci-e2e-test resourceVersion: "27813" uid: f46d5436-a1cc-4d47-ae27-936f230d4662 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:23:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-76a97-predictor namespace: kserve-ci-e2e-test resourceVersion: "27144" uid: beccadd8-e8df-49b5-9cdd-0dcd782dd527 kind: Event lastTimestamp: "2026-07-16T22:24:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:52Z" name: error-404-isvc-76a97-predictor.18c2e566ca0ed5f3 namespace: kserve-ci-e2e-test resourceVersion: "27814" uid: c67c22c3-f6a9-4c76-866e-34c7575b1b28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-76a97 namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: 2010ed32-c83d-45ed-86fc-6bc61e88100c kind: Event lastTimestamp: "2026-07-16T22:24:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:24:19Z" name: error-404-isvc-76a97.18c2e55fcaa665a8 namespace: kserve-ci-e2e-test resourceVersion: "27533" uid: 74a520c5-9520-4a85-b651-79cefb794444 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-76a97 namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: 2010ed32-c83d-45ed-86fc-6bc61e88100c kind: Event lastTimestamp: "2026-07-16T22:23:21Z" message: 'Failed to update status for InferenceService "error-404-isvc-76a97": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-76a97": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:23:21Z" name: error-404-isvc-76a97.18c2e55fcb5220b1 namespace: kserve-ci-e2e-test resourceVersion: "26950" uid: c204da4d-5e83-4b35-bc86-103f13ae3661 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-76a97 namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: 2010ed32-c83d-45ed-86fc-6bc61e88100c kind: Event lastTimestamp: "2026-07-16T22:23:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-76a97": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:23:21Z" name: error-404-isvc-76a97.18c2e55fcb528036 namespace: kserve-ci-e2e-test resourceVersion: "26951" uid: c430ec6f-9950-4059-8471-3466379235af reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-76a97 namespace: kserve-ci-e2e-test resourceVersion: "27530" uid: 2010ed32-c83d-45ed-86fc-6bc61e88100c kind: Event lastTimestamp: "2026-07-16T22:24:19Z" message: InferenceService [error-404-isvc-76a97] is Ready metadata: creationTimestamp: "2026-07-16T22:24:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:24:19Z" name: error-404-isvc-76a97.18c2e56d3325fb92 namespace: kserve-ci-e2e-test resourceVersion: "27531" uid: 9b32d8b4-9584-44bd-95c3-8aad3ba04346 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:02:37.414235Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z namespace: kserve-ci-e2e-test resourceVersion: "16374" uid: b2a4d504-f247-461b-9f65-b8a83e2837d6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-77f35-predictor-558485f7c5-gvp7z to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:02:37Z" name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z.18c2e43e04f28309 namespace: kserve-ci-e2e-test resourceVersion: "16383" uid: e6b9fd7f-113b-41b4-8b6a-887af6b26345 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: b2a4d504-f247-461b-9f65-b8a83e2837d6 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-77f35-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:37Z" name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z.18c2e43e13a74f1c namespace: kserve-ci-e2e-test resourceVersion: "16398" uid: 695efdba-055c-4113-bc3f-0f32e3a9e0e9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: b2a4d504-f247-461b-9f65-b8a83e2837d6 kind: Event lastTimestamp: "2026-07-16T22:02:38Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:02:38Z" name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z.18c2e43e43146684 namespace: kserve-ci-e2e-test resourceVersion: "16419" uid: 3bea7177-2b07-4b64-aee0-2e4e59df7f2a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: b2a4d504-f247-461b-9f65-b8a83e2837d6 kind: Event lastTimestamp: "2026-07-16T22:02:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:38Z" name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z.18c2e43e447ff052 namespace: kserve-ci-e2e-test resourceVersion: "16421" uid: 44e7befd-daab-49f0-9530-da464b91cfc7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: b2a4d504-f247-461b-9f65-b8a83e2837d6 kind: Event lastTimestamp: "2026-07-16T22:02:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:38Z" name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z.18c2e43e48485ce5 namespace: kserve-ci-e2e-test resourceVersion: "16422" uid: 2677cb7e-4806-419c-8865-1bbfd24bad05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: b2a4d504-f247-461b-9f65-b8a83e2837d6 kind: Event lastTimestamp: "2026-07-16T22:02:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:38Z" name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z.18c2e43e48c933b6 namespace: kserve-ci-e2e-test resourceVersion: "16423" uid: 6926e35d-4c64-4755-a94e-9d91e198b646 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: b2a4d504-f247-461b-9f65-b8a83e2837d6 kind: Event lastTimestamp: "2026-07-16T22:02:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:38Z" name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z.18c2e43e48d694d6 namespace: kserve-ci-e2e-test resourceVersion: "16424" uid: 9af9cd02-39a9-49e8-acdc-9c2856decc38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: b2a4d504-f247-461b-9f65-b8a83e2837d6 kind: Event lastTimestamp: "2026-07-16T22:02:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:38Z" name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z.18c2e43e4cb023a6 namespace: kserve-ci-e2e-test resourceVersion: "16426" uid: 53f249ac-fb7d-4803-848b-826b0b0959a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: b2a4d504-f247-461b-9f65-b8a83e2837d6 kind: Event lastTimestamp: "2026-07-16T22:02:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:38Z" name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z.18c2e43e4d327b02 namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 0af84d0a-d708-4e85-af22-08d07fc78663 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-16T22:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: b2a4d504-f247-461b-9f65-b8a83e2837d6 kind: Event lastTimestamp: "2026-07-16T22:11:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:54Z" name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z.18c2e43e58b81214 namespace: kserve-ci-e2e-test resourceVersion: "20742" uid: 0ef17faf-0b23-4414-8e0f-cb367ba18ec7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: b2a4d504-f247-461b-9f65-b8a83e2837d6 kind: Event lastTimestamp: "2026-07-16T22:11:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:51Z" name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z.18c2e4bf240a26e5 namespace: kserve-ci-e2e-test resourceVersion: "20605" uid: 0c10caa4-a38a-4ec6-9dc5-74f4671a07fd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: b2a4d504-f247-461b-9f65-b8a83e2837d6 kind: Event lastTimestamp: "2026-07-16T22:11:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:51Z" name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z.18c2e4bf240adc08 namespace: kserve-ci-e2e-test resourceVersion: "20608" uid: cd05d8ba-2657-40fb-8eb0-3eb4041c8a95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: b2a4d504-f247-461b-9f65-b8a83e2837d6 kind: Event lastTimestamp: "2026-07-16T22:11:54Z" message: 'Readiness probe failed: Get "https://10.134.0.44:8643/healthz": dial tcp 10.134.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:54Z" name: error-404-isvc-77f35-predictor-558485f7c5-gvp7z.18c2e4bfcd36217b namespace: kserve-ci-e2e-test resourceVersion: "20741" uid: c32efd96-79b0-4642-b07b-031e0ebbd73e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-77f35-predictor-558485f7c5 namespace: kserve-ci-e2e-test resourceVersion: "16358" uid: 82ef0050-a0de-44e7-8801-6d0db10deace kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: 'Created pod: error-404-isvc-77f35-predictor-558485f7c5-gvp7z' metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:37Z" name: error-404-isvc-77f35-predictor-558485f7c5.18c2e43e040fea61 namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: c66968bd-0020-4e96-86d7-0d26f665dd19 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-77f35-predictor namespace: kserve-ci-e2e-test resourceVersion: "16355" uid: b7482e74-f4cc-41ea-8f93-319f0699cd87 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: Scaled up replica set error-404-isvc-77f35-predictor-558485f7c5 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:37Z" name: error-404-isvc-77f35-predictor.18c2e43e021a5242 namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: 1660ce94-d9a9-4fb4-a674-6e688c8654a7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-77f35-predictor namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: b2777396-1662-4912-aee1-cb9918c62ed8 kind: Event lastTimestamp: "2026-07-16T22:02:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:52Z" name: error-404-isvc-77f35-predictor.18c2e441803a6fcf namespace: kserve-ci-e2e-test resourceVersion: "16578" uid: bb3a8e24-4091-45b7-b036-be003acc45bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-77f35-predictor namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: b2777396-1662-4912-aee1-cb9918c62ed8 kind: Event lastTimestamp: "2026-07-16T22:02:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:52Z" name: error-404-isvc-77f35-predictor.18c2e44180a9c797 namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: 0cb99469-a85c-41ce-8971-eb21c6bf5278 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:03:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-77f35-predictor namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: b2777396-1662-4912-aee1-cb9918c62ed8 kind: Event lastTimestamp: "2026-07-16T22:04:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:04:07Z" name: error-404-isvc-77f35-predictor.18c2e444ff7270c8 namespace: kserve-ci-e2e-test resourceVersion: "17248" uid: 7339082a-4a3f-43d8-9a15-5b87b3ac8a52 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:03:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-77f35-predictor namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: b2777396-1662-4912-aee1-cb9918c62ed8 kind: Event lastTimestamp: "2026-07-16T22:04:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:04:07Z" name: error-404-isvc-77f35-predictor.18c2e444ffe0b67a namespace: kserve-ci-e2e-test resourceVersion: "17249" uid: b209c6b0-5cf2-425d-ae58-406d1a30629a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-77f35 namespace: kserve-ci-e2e-test resourceVersion: "16339" uid: fcaca048-0dab-431e-bf6f-181f73ab658a kind: Event lastTimestamp: "2026-07-16T22:03:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:03:34Z" name: error-404-isvc-77f35.18c2e43e01bf3f35 namespace: kserve-ci-e2e-test resourceVersion: "16962" uid: 7f9d21a5-6133-4f59-ba75-62fc65500ce1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-77f35 namespace: kserve-ci-e2e-test resourceVersion: "16959" uid: fcaca048-0dab-431e-bf6f-181f73ab658a kind: Event lastTimestamp: "2026-07-16T22:03:34Z" message: InferenceService [error-404-isvc-77f35] is Ready metadata: creationTimestamp: "2026-07-16T22:03:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:03:34Z" name: error-404-isvc-77f35.18c2e44b68706741 namespace: kserve-ci-e2e-test resourceVersion: "16961" uid: 6b7506d1-2330-4a1d-8319-03b969016b9a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:22:37.630943Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-93462-predictor-655689f48c-4gb5h namespace: kserve-ci-e2e-test resourceVersion: "26384" uid: 97b91099-75f0-47a7-994a-fba7ebdfd400 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-93462-predictor-655689f48c-4gb5h to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:22:37Z" name: error-404-isvc-93462-predictor-655689f48c-4gb5h.18c2e55577701862 namespace: kserve-ci-e2e-test resourceVersion: "26389" uid: 407a4b5f-c0ef-4d92-b10d-50c0566cf9ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-93462-predictor-655689f48c-4gb5h namespace: kserve-ci-e2e-test resourceVersion: "26397" uid: 97b91099-75f0-47a7-994a-fba7ebdfd400 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: Add eth0 [10.134.0.56/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:22:38Z" name: error-404-isvc-93462-predictor-655689f48c-4gb5h.18c2e555921829d0 namespace: kserve-ci-e2e-test resourceVersion: "26408" uid: 393d7023-304a-4e49-99be-abc49ec13600 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-93462-predictor-655689f48c-4gb5h namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: 97b91099-75f0-47a7-994a-fba7ebdfd400 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:38Z" name: error-404-isvc-93462-predictor-655689f48c-4gb5h.18c2e55593c4735f namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: c15bf071-c85c-48fe-9c57-e1c08c908928 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-93462-predictor-655689f48c-4gb5h namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: 97b91099-75f0-47a7-994a-fba7ebdfd400 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:38Z" name: error-404-isvc-93462-predictor-655689f48c-4gb5h.18c2e55596f1c555 namespace: kserve-ci-e2e-test resourceVersion: "26411" uid: fb0b7e3a-d35b-4ba1-8b5d-ea3e72426bc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-93462-predictor-655689f48c-4gb5h namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: 97b91099-75f0-47a7-994a-fba7ebdfd400 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:38Z" name: error-404-isvc-93462-predictor-655689f48c-4gb5h.18c2e555977077f6 namespace: kserve-ci-e2e-test resourceVersion: "26412" uid: c7b937d0-5ffe-4dbf-bf47-620201e6e6f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-93462-predictor-655689f48c-4gb5h namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: 97b91099-75f0-47a7-994a-fba7ebdfd400 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:38Z" name: error-404-isvc-93462-predictor-655689f48c-4gb5h.18c2e555977bcb8d namespace: kserve-ci-e2e-test resourceVersion: "26413" uid: a028ca4c-6b8d-4038-931e-a7becdb0461b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-93462-predictor-655689f48c-4gb5h namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: 97b91099-75f0-47a7-994a-fba7ebdfd400 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:38Z" name: error-404-isvc-93462-predictor-655689f48c-4gb5h.18c2e5559b01b998 namespace: kserve-ci-e2e-test resourceVersion: "26416" uid: 86f0bbf3-3fdd-4c80-9a64-828fa6c3338c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-93462-predictor-655689f48c-4gb5h namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: 97b91099-75f0-47a7-994a-fba7ebdfd400 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:38Z" name: error-404-isvc-93462-predictor-655689f48c-4gb5h.18c2e5559b7c2208 namespace: kserve-ci-e2e-test resourceVersion: "26417" uid: 58c2e4b6-2c03-4738-80e1-399833dd218e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T22:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-93462-predictor-655689f48c-4gb5h namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: 97b91099-75f0-47a7-994a-fba7ebdfd400 kind: Event lastTimestamp: "2026-07-16T22:23:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.56:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:26Z" name: error-404-isvc-93462-predictor-655689f48c-4gb5h.18c2e556102385e6 namespace: kserve-ci-e2e-test resourceVersion: "27038" uid: 1149e24d-7d92-42d0-be24-185b7064c4f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:40:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-93462-predictor-655689f48c-4gb5h namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: 97b91099-75f0-47a7-994a-fba7ebdfd400 kind: Event lastTimestamp: "2026-07-16T22:40:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:40:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:40:07Z" name: error-404-isvc-93462-predictor-655689f48c-4gb5h.18c2e649ceedaac0 namespace: kserve-ci-e2e-test resourceVersion: "34160" uid: 7a0432b6-d82d-4c9c-bc54-201cb4ca035b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:40:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-93462-predictor-655689f48c-4gb5h namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: 97b91099-75f0-47a7-994a-fba7ebdfd400 kind: Event lastTimestamp: "2026-07-16T22:40:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:40:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:40:07Z" name: error-404-isvc-93462-predictor-655689f48c-4gb5h.18c2e649ceee523e namespace: kserve-ci-e2e-test resourceVersion: "34161" uid: 850fce50-7ee3-4697-a052-ba5de90b1c60 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-93462-predictor-655689f48c namespace: kserve-ci-e2e-test resourceVersion: "26373" uid: 4180d1b2-4cd3-41c2-b1cc-d30193d6960e kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: 'Created pod: error-404-isvc-93462-predictor-655689f48c-4gb5h' metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:22:37Z" name: error-404-isvc-93462-predictor-655689f48c.18c2e55576563198 namespace: kserve-ci-e2e-test resourceVersion: "26386" uid: 855f83c8-a37e-4357-9a2b-309bdb8e8ecc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-93462-predictor namespace: kserve-ci-e2e-test resourceVersion: "26368" uid: 18e45d56-7494-404c-a854-7031f11c0879 kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: Scaled up replica set error-404-isvc-93462-predictor-655689f48c from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:22:37Z" name: error-404-isvc-93462-predictor.18c2e55572fda339 namespace: kserve-ci-e2e-test resourceVersion: "26376" uid: b50f50d4-23dc-4678-8c6b-091d28b9adfc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:22:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-93462-predictor namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: f6fa5587-a758-4c6b-8ce8-52a43b84a984 kind: Event lastTimestamp: "2026-07-16T22:23:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:22:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:07Z" name: error-404-isvc-93462-predictor.18c2e558f1c55559 namespace: kserve-ci-e2e-test resourceVersion: "26704" uid: 7507bbac-6c27-4de3-b9fa-aa912d163b70 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:22:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-93462-predictor namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: f6fa5587-a758-4c6b-8ce8-52a43b84a984 kind: Event lastTimestamp: "2026-07-16T22:23:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:22:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:07Z" name: error-404-isvc-93462-predictor.18c2e558f24325fe namespace: kserve-ci-e2e-test resourceVersion: "26705" uid: 0a69bb5b-091b-461d-aadb-00e6344dd2df reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-93462-predictor namespace: kserve-ci-e2e-test resourceVersion: "26571" uid: f6fa5587-a758-4c6b-8ce8-52a43b84a984 kind: Event lastTimestamp: "2026-07-16T22:24:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:22Z" name: error-404-isvc-93462-predictor.18c2e55ff03d032f namespace: kserve-ci-e2e-test resourceVersion: "27585" uid: 5416a6dc-aa4b-48b7-963b-0a4381a4e3ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-93462-predictor namespace: kserve-ci-e2e-test resourceVersion: "26571" uid: f6fa5587-a758-4c6b-8ce8-52a43b84a984 kind: Event lastTimestamp: "2026-07-16T22:24:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:22Z" name: error-404-isvc-93462-predictor.18c2e55ff0b0ed4b namespace: kserve-ci-e2e-test resourceVersion: "27586" uid: 4acca307-b911-4117-8abc-a579fc2bfca4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-93462 namespace: kserve-ci-e2e-test resourceVersion: "26347" uid: 495483d8-4f22-45c2-b3a8-7d2ccc53ac82 kind: Event lastTimestamp: "2026-07-16T22:23:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:23:36Z" name: error-404-isvc-93462.18c2e555734157c7 namespace: kserve-ci-e2e-test resourceVersion: "27131" uid: 6486916e-70b1-44c4-ac3a-21d91a12d959 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-93462 namespace: kserve-ci-e2e-test resourceVersion: "26347" uid: 495483d8-4f22-45c2-b3a8-7d2ccc53ac82 kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-93462": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-93462": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:22:37Z" name: error-404-isvc-93462.18c2e55574ac7efb namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: e1a4bd4c-f4cb-410a-bf6c-f8d1fef1b066 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-93462 namespace: kserve-ci-e2e-test resourceVersion: "26347" uid: 495483d8-4f22-45c2-b3a8-7d2ccc53ac82 kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-93462": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:22:37Z" name: error-404-isvc-93462.18c2e55574ace89a namespace: kserve-ci-e2e-test resourceVersion: "26385" uid: 8f345c0b-e13a-4f32-b1a0-83aa87fbbb07 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-93462 namespace: kserve-ci-e2e-test resourceVersion: "27129" uid: 495483d8-4f22-45c2-b3a8-7d2ccc53ac82 kind: Event lastTimestamp: "2026-07-16T22:23:36Z" message: InferenceService [error-404-isvc-93462] is Ready metadata: creationTimestamp: "2026-07-16T22:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:23:36Z" name: error-404-isvc-93462.18c2e563202f76a9 namespace: kserve-ci-e2e-test resourceVersion: "27130" uid: d2f2ca95-b7f0-4816-8261-1f3b60177b62 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:13:12.603755Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66 namespace: kserve-ci-e2e-test resourceVersion: "21723" uid: 6591ac58-f4a1-4d2b-ab1e-653071b8b3cd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66 to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:13:12Z" name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66.18c2e4d1e93167f7 namespace: kserve-ci-e2e-test resourceVersion: "21727" uid: cce931f7-8586-4274-b01d-67b58cc022d3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 6591ac58-f4a1-4d2b-ab1e-653071b8b3cd kind: Event lastTimestamp: "2026-07-16T22:13:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a3a91-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:13Z" name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66.18c2e4d1f65e6ba1 namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: 19f6086a-7f87-49de-bb1f-e173988aef51 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66 namespace: kserve-ci-e2e-test resourceVersion: "21734" uid: 6591ac58-f4a1-4d2b-ab1e-653071b8b3cd kind: Event lastTimestamp: "2026-07-16T22:13:14Z" message: Add eth0 [10.134.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:13:14Z" name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66.18c2e4d25e1de9fa namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: 9b0c2935-cd8f-4032-b61f-e18d283bec27 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 6591ac58-f4a1-4d2b-ab1e-653071b8b3cd kind: Event lastTimestamp: "2026-07-16T22:13:14Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:14Z" name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66.18c2e4d25fb056a4 namespace: kserve-ci-e2e-test resourceVersion: "21777" uid: 0af2a3ac-4d12-4d38-9152-4b2ae97c12dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 6591ac58-f4a1-4d2b-ab1e-653071b8b3cd kind: Event lastTimestamp: "2026-07-16T22:13:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:14Z" name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66.18c2e4d26352959b namespace: kserve-ci-e2e-test resourceVersion: "21778" uid: 1a7ddcce-069b-4fb4-8cd2-22b3b556e35e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 6591ac58-f4a1-4d2b-ab1e-653071b8b3cd kind: Event lastTimestamp: "2026-07-16T22:13:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:14Z" name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66.18c2e4d263ceb949 namespace: kserve-ci-e2e-test resourceVersion: "21779" uid: 44f9e31c-3099-491a-ba85-c2715c04f33a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 6591ac58-f4a1-4d2b-ab1e-653071b8b3cd kind: Event lastTimestamp: "2026-07-16T22:13:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:14Z" name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66.18c2e4d263d9c6db namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: 05d168ba-6876-4a9e-9377-9eaa0e4d84d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 6591ac58-f4a1-4d2b-ab1e-653071b8b3cd kind: Event lastTimestamp: "2026-07-16T22:13:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:14Z" name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66.18c2e4d26767f7d1 namespace: kserve-ci-e2e-test resourceVersion: "21781" uid: 6978c360-600c-407a-9e7b-42fb6a6dfd8c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 6591ac58-f4a1-4d2b-ab1e-653071b8b3cd kind: Event lastTimestamp: "2026-07-16T22:13:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:14Z" name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66.18c2e4d267efe7ec namespace: kserve-ci-e2e-test resourceVersion: "21782" uid: 6fd5cd6e-7cef-4eee-82b9-93052e381a9a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T22:13:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 6591ac58-f4a1-4d2b-ab1e-653071b8b3cd kind: Event lastTimestamp: "2026-07-16T22:14:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.52:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:13:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:14:02Z" name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66.18c2e4d2be302025 namespace: kserve-ci-e2e-test resourceVersion: "22232" uid: 7b21131e-9a27-43e9-97b8-af4a8acf2ea3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 6591ac58-f4a1-4d2b-ab1e-653071b8b3cd kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:37Z" name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66.18c2e5557371da77 namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: ac3548b8-7e81-4c27-9cb3-990b971fbbed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 6591ac58-f4a1-4d2b-ab1e-653071b8b3cd kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:37Z" name: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66.18c2e55573726cfb namespace: kserve-ci-e2e-test resourceVersion: "26381" uid: 928d4012-c22a-4ae0-b023-9f12e0cc07b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a3a91-predictor-844ff5cbfc namespace: kserve-ci-e2e-test resourceVersion: "21713" uid: d7d6448e-7451-4f44-908f-270082453bf2 kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: 'Created pod: error-404-isvc-a3a91-predictor-844ff5cbfc-fnf66' metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:12Z" name: error-404-isvc-a3a91-predictor-844ff5cbfc.18c2e4d1e8b3c397 namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: ec298ab0-b4c1-46f8-b559-d899289e2c08 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a3a91-predictor namespace: kserve-ci-e2e-test resourceVersion: "21711" uid: 6815e565-6466-4508-a7f4-4630474aa003 kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: Scaled up replica set error-404-isvc-a3a91-predictor-844ff5cbfc from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:12Z" name: error-404-isvc-a3a91-predictor.18c2e4d1e6b59a2b namespace: kserve-ci-e2e-test resourceVersion: "21716" uid: 4cab0757-ee6f-4d35-9e20-ed9073558811 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:13:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3a91-predictor namespace: kserve-ci-e2e-test resourceVersion: "21712" uid: 802b1816-3e8d-412a-a9bb-79c825bfb67d kind: Event lastTimestamp: "2026-07-16T22:14:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:13:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:27Z" name: error-404-isvc-a3a91-predictor.18c2e4d5655e3006 namespace: kserve-ci-e2e-test resourceVersion: "22476" uid: 9579413b-6202-462c-a65d-80fe41a1db91 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:13:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3a91-predictor namespace: kserve-ci-e2e-test resourceVersion: "21712" uid: 802b1816-3e8d-412a-a9bb-79c825bfb67d kind: Event lastTimestamp: "2026-07-16T22:14:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:13:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:27Z" name: error-404-isvc-a3a91-predictor.18c2e4d565ebc88c namespace: kserve-ci-e2e-test resourceVersion: "22477" uid: 57fdbd9f-d474-43b5-bb87-60b5d5a01de3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3a91-predictor namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: 802b1816-3e8d-412a-a9bb-79c825bfb67d kind: Event lastTimestamp: "2026-07-16T22:13:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:13:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:42Z" name: error-404-isvc-a3a91-predictor.18c2e4d8e4aa98d0 namespace: kserve-ci-e2e-test resourceVersion: "22047" uid: 20b2c9a1-31d5-4d74-b0e7-e535b597b9dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3a91-predictor namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: 802b1816-3e8d-412a-a9bb-79c825bfb67d kind: Event lastTimestamp: "2026-07-16T22:13:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:13:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:42Z" name: error-404-isvc-a3a91-predictor.18c2e4d8e4fa3987 namespace: kserve-ci-e2e-test resourceVersion: "22049" uid: 198a2e6f-800d-4179-b86a-427816ed08eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: ac907a5f-2bf6-4da5-b690-a6cdffb413fb kind: Event lastTimestamp: "2026-07-16T22:14:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:14:12Z" name: error-404-isvc-a3a91.18c2e4d1e6c0040b namespace: kserve-ci-e2e-test resourceVersion: "22328" uid: c0be1526-df44-40b6-9949-214ebf181ce0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: ac907a5f-2bf6-4da5-b690-a6cdffb413fb kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-a3a91": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a3a91": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:13:12Z" name: error-404-isvc-a3a91.18c2e4d1e7445406 namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: c794aa00-6ba6-44c5-81f3-7b3df835a71b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: ac907a5f-2bf6-4da5-b690-a6cdffb413fb kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a3a91": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:13:12Z" name: error-404-isvc-a3a91.18c2e4d1e744ac96 namespace: kserve-ci-e2e-test resourceVersion: "21720" uid: 12f0341f-6816-4342-ae48-38791fe174a8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "22325" uid: ac907a5f-2bf6-4da5-b690-a6cdffb413fb kind: Event lastTimestamp: "2026-07-16T22:14:12Z" message: InferenceService [error-404-isvc-a3a91] is Ready metadata: creationTimestamp: "2026-07-16T22:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:14:12Z" name: error-404-isvc-a3a91.18c2e4dfce9cd12b namespace: kserve-ci-e2e-test resourceVersion: "22327" uid: 2cc862af-7d92-4402-aca7-60612d0014d1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:22:01.615239Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: 653062fd-7a45-4f57-a939-69400a3be794 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a3d07-predictor-f5d955f89-cthdq to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:22:01Z" name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq.18c2e54d14bc1570 namespace: kserve-ci-e2e-test resourceVersion: "25964" uid: 641de726-90d3-4ec5-9717-f661ed827d23 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: 653062fd-7a45-4f57-a939-69400a3be794 kind: Event lastTimestamp: "2026-07-16T22:22:02Z" message: Add eth0 [10.134.0.54/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:22:02Z" name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq.18c2e54d304a1bd5 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: c0b6c52f-9292-43e2-bddd-22d0d1c66613 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 653062fd-7a45-4f57-a939-69400a3be794 kind: Event lastTimestamp: "2026-07-16T22:22:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:02Z" name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq.18c2e54d31f25041 namespace: kserve-ci-e2e-test resourceVersion: "25985" uid: 6b26983f-12ae-4d5b-9bfc-6713c15a58ab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 653062fd-7a45-4f57-a939-69400a3be794 kind: Event lastTimestamp: "2026-07-16T22:22:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:02Z" name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq.18c2e54d35352084 namespace: kserve-ci-e2e-test resourceVersion: "25988" uid: 98be9188-649d-446b-b080-fd263dcd1e4a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 653062fd-7a45-4f57-a939-69400a3be794 kind: Event lastTimestamp: "2026-07-16T22:22:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:02Z" name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq.18c2e54d35b1dfc4 namespace: kserve-ci-e2e-test resourceVersion: "25989" uid: 15a0a812-717f-4121-8b4c-56fe46a128c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 653062fd-7a45-4f57-a939-69400a3be794 kind: Event lastTimestamp: "2026-07-16T22:22:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:02Z" name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq.18c2e54d35bc3c26 namespace: kserve-ci-e2e-test resourceVersion: "25990" uid: 6ecc8295-6fbc-444a-a7b6-6146366bf0f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 653062fd-7a45-4f57-a939-69400a3be794 kind: Event lastTimestamp: "2026-07-16T22:22:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:02Z" name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq.18c2e54d39415d47 namespace: kserve-ci-e2e-test resourceVersion: "25992" uid: e9f74109-17e1-44d3-85f9-af66a2daeaca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 653062fd-7a45-4f57-a939-69400a3be794 kind: Event lastTimestamp: "2026-07-16T22:22:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:02Z" name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq.18c2e54d39c33861 namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: d72a738b-ceb0-4c0c-881c-1d5840fc6abd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T22:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 653062fd-7a45-4f57-a939-69400a3be794 kind: Event lastTimestamp: "2026-07-16T22:22:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.54:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:48Z" name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq.18c2e54d651a770a namespace: kserve-ci-e2e-test resourceVersion: "26537" uid: d954a5a6-a7d7-453b-b697-71184c926c16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 653062fd-7a45-4f57-a939-69400a3be794 kind: Event lastTimestamp: "2026-07-16T22:23:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:21Z" name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq.18c2e55fca9621b8 namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 25db0640-dd15-4848-9301-0a32b2742c91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 653062fd-7a45-4f57-a939-69400a3be794 kind: Event lastTimestamp: "2026-07-16T22:23:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:21Z" name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq.18c2e55fca9ab4a3 namespace: kserve-ci-e2e-test resourceVersion: "26948" uid: 19c8bf9f-d0ff-4acd-bf78-0006dea6cbef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 653062fd-7a45-4f57-a939-69400a3be794 kind: Event lastTimestamp: "2026-07-16T22:23:23Z" message: 'Readiness probe failed: Get "https://10.134.0.54:8643/healthz": dial tcp 10.134.0.54:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:23:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:23Z" name: error-404-isvc-a3d07-predictor-f5d955f89-cthdq.18c2e5604174b87a namespace: kserve-ci-e2e-test resourceVersion: "27015" uid: 37e10233-0a9c-43ac-8c7c-1a518dce242f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a3d07-predictor-f5d955f89 namespace: kserve-ci-e2e-test resourceVersion: "25945" uid: 2defddcf-085c-46b1-bf5c-e1c8c81da927 kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: 'Created pod: error-404-isvc-a3d07-predictor-f5d955f89-cthdq' metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:22:01Z" name: error-404-isvc-a3d07-predictor-f5d955f89.18c2e54d138c9ff1 namespace: kserve-ci-e2e-test resourceVersion: "25959" uid: e2ce4bb1-89a2-4750-9ddf-64563eba0f55 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a3d07-predictor namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 85335999-03ae-48f4-ac48-fba86957ba45 kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: Scaled up replica set error-404-isvc-a3d07-predictor-f5d955f89 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:22:01Z" name: error-404-isvc-a3d07-predictor.18c2e54d109fa459 namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: af36a585-a0cb-4dbd-a14e-b3c5fefbf0a7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:22:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3d07-predictor namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 475aac4d-e837-4861-9d04-406554a878b4 kind: Event lastTimestamp: "2026-07-16T22:22:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:22:31Z" name: error-404-isvc-a3d07-predictor.18c2e5508ef1c460 namespace: kserve-ci-e2e-test resourceVersion: "26247" uid: 7a982d71-2552-4edf-b54e-652889844252 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:22:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3d07-predictor namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: 475aac4d-e837-4861-9d04-406554a878b4 kind: Event lastTimestamp: "2026-07-16T22:22:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:22:31Z" name: error-404-isvc-a3d07-predictor.18c2e5508f696e7b namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: b8af6bf5-a1f4-4d05-8c4d-dc7a2d1f260b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:22:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3d07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 475aac4d-e837-4861-9d04-406554a878b4 kind: Event lastTimestamp: "2026-07-16T22:23:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:16Z" name: error-404-isvc-a3d07-predictor.18c2e5578cf1024e namespace: kserve-ci-e2e-test resourceVersion: "26818" uid: c39e5877-9862-4d4b-99c0-2102bdbb6020 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:22:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3d07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 475aac4d-e837-4861-9d04-406554a878b4 kind: Event lastTimestamp: "2026-07-16T22:23:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:16Z" name: error-404-isvc-a3d07-predictor.18c2e5578d7ace92 namespace: kserve-ci-e2e-test resourceVersion: "26820" uid: a5f877c1-6ea6-4a7a-8a31-188cdf10c443 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3d07 namespace: kserve-ci-e2e-test resourceVersion: "25916" uid: 531258b8-bbc7-44fc-ac34-8f7881959c9c kind: Event lastTimestamp: "2026-07-16T22:22:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:22:59Z" name: error-404-isvc-a3d07.18c2e54d10935104 namespace: kserve-ci-e2e-test resourceVersion: "26630" uid: c3502bfa-76e7-4261-811d-6bd73e0a1da6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3d07 namespace: kserve-ci-e2e-test resourceVersion: "25916" uid: 531258b8-bbc7-44fc-ac34-8f7881959c9c kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: 'Failed to update status for InferenceService "error-404-isvc-a3d07": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a3d07": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:22:01Z" name: error-404-isvc-a3d07.18c2e54d112535ef namespace: kserve-ci-e2e-test resourceVersion: "25951" uid: 9bd3382b-b0ae-4724-8ac3-593fd34f7259 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3d07 namespace: kserve-ci-e2e-test resourceVersion: "25916" uid: 531258b8-bbc7-44fc-ac34-8f7881959c9c kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a3d07": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:22:01Z" name: error-404-isvc-a3d07.18c2e54d11258f6a namespace: kserve-ci-e2e-test resourceVersion: "25953" uid: af84c645-29e3-42c6-9765-fb8c2c08f335 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3d07 namespace: kserve-ci-e2e-test resourceVersion: "26628" uid: 531258b8-bbc7-44fc-ac34-8f7881959c9c kind: Event lastTimestamp: "2026-07-16T22:22:59Z" message: InferenceService [error-404-isvc-a3d07] is Ready metadata: creationTimestamp: "2026-07-16T22:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:22:59Z" name: error-404-isvc-a3d07.18c2e55a75116be6 namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: a63ede31-6a23-4258-9c87-5c6b58a24bc7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:12:36.650999Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5 namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: 86a47779-330b-4d20-87ba-123ae324eea8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c1850-predictor-597c44d6bf-wwcv5 to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:12:36Z" name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5.18c2e4c98a3de29b namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: c868b3a7-bf8e-481c-b84a-8e828bafdeef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5 namespace: kserve-ci-e2e-test resourceVersion: "21241" uid: 86a47779-330b-4d20-87ba-123ae324eea8 kind: Event lastTimestamp: "2026-07-16T22:12:37Z" message: Add eth0 [10.134.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:12:37Z" name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5.18c2e4c9a5c78e25 namespace: kserve-ci-e2e-test resourceVersion: "21260" uid: d6ab43ab-0d70-4ad5-9d9c-e4fa75e6974f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 86a47779-330b-4d20-87ba-123ae324eea8 kind: Event lastTimestamp: "2026-07-16T22:12:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:37Z" name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5.18c2e4c9a74c6bcf namespace: kserve-ci-e2e-test resourceVersion: "21263" uid: 38cd2502-dd55-4506-9672-5680e24756b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 86a47779-330b-4d20-87ba-123ae324eea8 kind: Event lastTimestamp: "2026-07-16T22:12:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:37Z" name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5.18c2e4c9ab33f166 namespace: kserve-ci-e2e-test resourceVersion: "21267" uid: 78df3ca2-0cfc-4f48-832c-384db95d8569 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 86a47779-330b-4d20-87ba-123ae324eea8 kind: Event lastTimestamp: "2026-07-16T22:12:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:37Z" name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5.18c2e4c9ac21410c namespace: kserve-ci-e2e-test resourceVersion: "21268" uid: 591ac714-04f0-4f25-bc83-17e1b2ea79c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 86a47779-330b-4d20-87ba-123ae324eea8 kind: Event lastTimestamp: "2026-07-16T22:12:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:37Z" name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5.18c2e4c9ac30e6dc namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: 5306fcfd-9f22-4444-92e6-00d71a1d1e32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 86a47779-330b-4d20-87ba-123ae324eea8 kind: Event lastTimestamp: "2026-07-16T22:12:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:37Z" name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5.18c2e4c9af8bbbbe namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: fd548a87-7175-4c42-a93b-d510fbe03b88 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 86a47779-330b-4d20-87ba-123ae324eea8 kind: Event lastTimestamp: "2026-07-16T22:12:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:37Z" name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5.18c2e4c9b00d4b91 namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 2741da88-faab-4ac0-8756-0bb705687e88 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 86a47779-330b-4d20-87ba-123ae324eea8 kind: Event lastTimestamp: "2026-07-16T22:22:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.50:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:03Z" name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5.18c2e4c9d7ad291a namespace: kserve-ci-e2e-test resourceVersion: "26015" uid: 84c25870-59b2-4cdd-a93f-4b94d36e8940 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 86a47779-330b-4d20-87ba-123ae324eea8 kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:01Z" name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5.18c2e54d11aba6fb namespace: kserve-ci-e2e-test resourceVersion: "25952" uid: 18bd6e87-397c-4907-8649-a5964dafc473 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 86a47779-330b-4d20-87ba-123ae324eea8 kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:01Z" name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5.18c2e54d11ac2510 namespace: kserve-ci-e2e-test resourceVersion: "25954" uid: c9383bdb-5e25-4016-b09d-511108bf301b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 86a47779-330b-4d20-87ba-123ae324eea8 kind: Event lastTimestamp: "2026-07-16T22:22:03Z" message: 'Readiness probe failed: Get "https://10.134.0.50:8643/healthz": dial tcp 10.134.0.50:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:22:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:03Z" name: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5.18c2e54da04d9cec namespace: kserve-ci-e2e-test resourceVersion: "26014" uid: e801044f-0767-4642-9581-b5641a3782fd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c1850-predictor-597c44d6bf namespace: kserve-ci-e2e-test resourceVersion: "21224" uid: afc2416e-d202-4456-b369-f60a7d65599e kind: Event lastTimestamp: "2026-07-16T22:12:36Z" message: 'Created pod: error-404-isvc-c1850-predictor-597c44d6bf-wwcv5' metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:36Z" name: error-404-isvc-c1850-predictor-597c44d6bf.18c2e4c989d26d4d namespace: kserve-ci-e2e-test resourceVersion: "21234" uid: f130b340-d5fb-4b0d-9c1f-1931b95ded5b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c1850-predictor namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: c729fb66-509a-40a6-b087-bad7c8e896f8 kind: Event lastTimestamp: "2026-07-16T22:12:36Z" message: Scaled up replica set error-404-isvc-c1850-predictor-597c44d6bf from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:36Z" name: error-404-isvc-c1850-predictor.18c2e4c988054c2d namespace: kserve-ci-e2e-test resourceVersion: "21226" uid: dd024d46-8393-4986-86f8-93e1d9dcd49b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:12:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1850-predictor namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 21169c6d-1680-4745-8a05-7b9c964024dc kind: Event lastTimestamp: "2026-07-16T22:13:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:06Z" name: error-404-isvc-c1850-predictor.18c2e4cd06c769c0 namespace: kserve-ci-e2e-test resourceVersion: "21579" uid: bab72191-41bb-40ed-b73d-c25f3e468447 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:12:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1850-predictor namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 21169c6d-1680-4745-8a05-7b9c964024dc kind: Event lastTimestamp: "2026-07-16T22:13:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:06Z" name: error-404-isvc-c1850-predictor.18c2e4cd0736fd58 namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: fa0451cf-3350-492d-a2f5-8499ab70d5c5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1850-predictor namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 21169c6d-1680-4745-8a05-7b9c964024dc kind: Event lastTimestamp: "2026-07-16T22:14:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:06Z" name: error-404-isvc-c1850-predictor.18c2e4d4057b223c namespace: kserve-ci-e2e-test resourceVersion: "22270" uid: 666f1b3a-98d6-4376-a30e-906300ec5042 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1850-predictor namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 21169c6d-1680-4745-8a05-7b9c964024dc kind: Event lastTimestamp: "2026-07-16T22:14:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:06Z" name: error-404-isvc-c1850-predictor.18c2e4d4062f4eb9 namespace: kserve-ci-e2e-test resourceVersion: "22271" uid: 359012dc-7eb8-46be-9201-bf8ef278a179 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1850 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 22957fdb-210a-4e42-8082-f3bd51f10248 kind: Event lastTimestamp: "2026-07-16T22:13:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:13:34Z" name: error-404-isvc-c1850.18c2e4c9881822f3 namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: 646ccd26-97e0-4396-9084-ef3d976bb73b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1850 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 22957fdb-210a-4e42-8082-f3bd51f10248 kind: Event lastTimestamp: "2026-07-16T22:12:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-c1850": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c1850": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:12:36Z" name: error-404-isvc-c1850.18c2e4c988971bd9 namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: d77c401a-42db-40d0-8005-d87eff416aae reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1850 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 22957fdb-210a-4e42-8082-f3bd51f10248 kind: Event lastTimestamp: "2026-07-16T22:12:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c1850": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:12:36Z" name: error-404-isvc-c1850.18c2e4c988977278 namespace: kserve-ci-e2e-test resourceVersion: "21230" uid: 5377eca2-57a2-4adb-bc4d-8d15e783d37d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1850 namespace: kserve-ci-e2e-test resourceVersion: "21958" uid: 22957fdb-210a-4e42-8082-f3bd51f10248 kind: Event lastTimestamp: "2026-07-16T22:13:34Z" message: InferenceService [error-404-isvc-c1850] is Ready metadata: creationTimestamp: "2026-07-16T22:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:13:34Z" name: error-404-isvc-c1850.18c2e4d6e792aeab namespace: kserve-ci-e2e-test resourceVersion: "21959" uid: d25c3b7f-5adb-4ad8-bfe1-549799464bfc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:00:17.247864Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d9a7f-predictor-54f6744545-sj44g to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:00:17Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e41d6261592f namespace: kserve-ci-e2e-test resourceVersion: "14691" uid: 1c05b875-8db8-4c24-a06e-d2c45de19e79 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14695" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:00:17Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e41d7d755aad namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 2966fbfd-bf08-4ac6-a381-8022116a6074 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1745" metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:17Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e41d7f3b5093 namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: 9f9c2ed3-8cc9-416d-a14a-dd6d6f671f50 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: "2026-07-16T22:00:33Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1745" in 15.661s (15.661s including waiting). Image size: 1334622813 bytes.' metadata: creationTimestamp: "2026-07-16T22:00:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:33Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e42124b5930e namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: a4e26e22-406b-4877-b5b9-6037db213206 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: "2026-07-16T22:00:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:00:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:33Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e421287da0b0 namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: 048d3dbd-85e9-42b8-b036-b33bcdfeb697 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: "2026-07-16T22:00:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:00:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:33Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e421291facbc namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 54f5ee10-9cce-42e5-a0fe-de16f1968616 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: "2026-07-16T22:00:33Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-16T22:00:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:33Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e421292ea87f namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: e12a4c66-d4ad-4770-83b4-08a99d716d99 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: "2026-07-16T22:00:36Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.58s (2.58s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-16T22:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:36Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e421c2fddbb3 namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: 2a79c340-372e-4df0-be80-b179089cafa0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: "2026-07-16T22:00:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:36Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e421c6ab4419 namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: 620d86cc-bc2b-4151-82fb-4fb9f41cbd5e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: "2026-07-16T22:00:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:36Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e421c7343b66 namespace: kserve-ci-e2e-test resourceVersion: "14975" uid: 3ef865ba-b1aa-403d-8a3d-07a22b552f9c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T22:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: "2026-07-16T22:01:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:53Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e422052a0e71 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 204e63ed-313c-4c75-a98a-1d52ec974a03 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:51Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e4334c581e7a namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: d93bdb11-6caa-4cab-aefe-7589efd8b0fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:51Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e4334c589f69 namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: f5766e15-541b-48b8-a3da-e7c164093588 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 9a49da02-ab1e-47aa-8a44-de3f7311e4d3 kind: Event lastTimestamp: "2026-07-16T22:01:53Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:53Z" name: error-404-isvc-d9a7f-predictor-54f6744545-sj44g.18c2e433b768bdbe namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: f95900df-de57-4643-9c58-38f8fd8a04bc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d9a7f-predictor-54f6744545 namespace: kserve-ci-e2e-test resourceVersion: "14677" uid: 6dde7c5e-271b-4a8b-a600-de41c49d2119 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: 'Created pod: error-404-isvc-d9a7f-predictor-54f6744545-sj44g' metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:00:17Z" name: error-404-isvc-d9a7f-predictor-54f6744545.18c2e41d60c56d23 namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 3e4725f0-63a4-4bec-805e-471954b34f29 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d9a7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: d07187e2-158c-46aa-ba7f-f2c268db00d1 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: Scaled up replica set error-404-isvc-d9a7f-predictor-54f6744545 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:00:17Z" name: error-404-isvc-d9a7f-predictor.18c2e41d5bc56dfd namespace: kserve-ci-e2e-test resourceVersion: "14678" uid: 1ea8713f-b7cf-417d-89ba-42f6052d42db reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:00:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d9a7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: 2a28d6ea-8e65-4cc8-a9ba-ab751568a4f8 kind: Event lastTimestamp: "2026-07-16T22:01:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:00:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:02Z" name: error-404-isvc-d9a7f-predictor.18c2e420da3c67b7 namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: 8f1e2709-e87e-4dd1-aead-886825888dc2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:00:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d9a7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: 2a28d6ea-8e65-4cc8-a9ba-ab751568a4f8 kind: Event lastTimestamp: "2026-07-16T22:01:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:00:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:02Z" name: error-404-isvc-d9a7f-predictor.18c2e420dab146eb namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: 426b97e6-8141-46dc-a5e3-ea6f151d9dfc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:01:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d9a7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2a28d6ea-8e65-4cc8-a9ba-ab751568a4f8 kind: Event lastTimestamp: "2026-07-16T22:01:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:01:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:47Z" name: error-404-isvc-d9a7f-predictor.18c2e42b57a136ab namespace: kserve-ci-e2e-test resourceVersion: "15646" uid: c86658a0-7cce-4a10-b6a4-636379e5ed74 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:01:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d9a7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 2a28d6ea-8e65-4cc8-a9ba-ab751568a4f8 kind: Event lastTimestamp: "2026-07-16T22:01:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:01:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:47Z" name: error-404-isvc-d9a7f-predictor.18c2e42b587636f0 namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: e051ca37-c600-46ae-bd53-4f032054ddc8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d9a7f namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 66c1c4a5-e012-4b70-be6a-41758780b43b kind: Event lastTimestamp: "2026-07-16T22:01:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:01:23Z" name: error-404-isvc-d9a7f.18c2e41d5c1a9496 namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 8c480e4e-b715-41b7-ac01-4397d0b9483d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d9a7f namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 66c1c4a5-e012-4b70-be6a-41758780b43b kind: Event lastTimestamp: "2026-07-16T22:01:23Z" message: InferenceService [error-404-isvc-d9a7f] is Ready metadata: creationTimestamp: "2026-07-16T22:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:01:23Z" name: error-404-isvc-d9a7f.18c2e42cc0c7d42f namespace: kserve-ci-e2e-test resourceVersion: "15409" uid: 97ef518f-92cc-415b-93d1-bc9897d82ed3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:00:17.024644Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14661" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-57c485b568-mrdqg to ip-10-0-128-250.ec2.internal metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:00:17Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e41d55134feb namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: 8d266af4-6d50-435f-8b1c-2f6a7002415b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:00:17Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e41d6e66d87b namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: c3b61e6f-a1c3-400b-a38b-35e2849eefda reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5d63c390841fcd6358d7b4068789845000a8eff3dfe19f0f9b697dde4d31930a" already present on machine metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:17Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e41d70a85265 namespace: kserve-ci-e2e-test resourceVersion: "14716" uid: 45df7823-3172-4b66-ae46-54d8d2cc590c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:17Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e41d73d6b255 namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: 44c7ff2a-04cc-4986-a62d-ca1f60edbd2f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:17Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e41d74487aab namespace: kserve-ci-e2e-test resourceVersion: "14733" uid: 5b0a757c-b516-40b2-9f60-2184be2a6d34 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:00:21Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1745" metadata: creationTimestamp: "2026-07-16T22:00:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:21Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e41e76ba5ec5 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 77b5153d-d306-494c-9d26-2b84aeea3e09 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:00:49Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1745" in 27.574s (27.574s including waiting). Image size: 1555872170 bytes.' metadata: creationTimestamp: "2026-07-16T22:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:49Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e424e2453ae7 namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: e5669259-06ef-43ea-93b5-54d7fbe1a17e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:00:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:49Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e424e5b95e0e namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: a2b352d9-19a9-4623-957b-0404e624dd3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:00:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:49Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e424e62bd163 namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: d8af0032-682a-43af-969f-e7de56aad3e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:00:49Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-16T22:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:49Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e424e63a4019 namespace: kserve-ci-e2e-test resourceVersion: "15097" uid: ceb3b300-837f-41df-b3ee-79a0d1c59ce1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:00:52Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.902s (2.902s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-16T22:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:52Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e42593393aba namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: c96b24d2-8e43-4218-9fc5-363772896a19 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:00:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:52Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e42597c6452e namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: 9962dbf3-1fd6-45bb-bed6-8fd0ca013652 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:00:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:52Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e4259841f12f namespace: kserve-ci-e2e-test resourceVersion: "15131" uid: 6cb67424-8ddd-43d9-8504-74839deb4169 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-16T22:00:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:01:59Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:00:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:59Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e425bbfb09fa namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: c6648540-804f-4475-8277-606b6c4d655b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:37Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e43df93138ac namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: d42f2723-68ec-4998-bf86-4f95cf26952c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:37Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e43df931d734 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: d143633e-6fe6-4b7f-a60b-2143120baf3e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 96eeb52b-4743-47a6-84b5-c206d12018f5 kind: Event lastTimestamp: "2026-07-16T22:02:39Z" message: 'Readiness probe failed: Get "https://10.132.0.17:8643/healthz": dial tcp 10.132.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:39Z" name: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg.18c2e43e6a87250c namespace: kserve-ci-e2e-test resourceVersion: "16439" uid: 26a4b972-a029-4a3a-a0a4-0678a4e233d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-57c485b568 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: ca6f7bcf-1d85-4807-8407-647efd95bf84 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-57c485b568-mrdqg' metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:00:17Z" name: isvc-sklearn-graph-1-predictor-57c485b568.18c2e41d54877ac6 namespace: kserve-ci-e2e-test resourceVersion: "14663" uid: ad72d970-ee40-4597-9695-ebbbedf1e369 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: 42a4304b-a50d-46c0-8ca7-c05818a52c9f kind: Event lastTimestamp: "2026-07-16T22:00:16Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-57c485b568 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:00:16Z" name: isvc-sklearn-graph-1-predictor.18c2e41d5154ef0c namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 49f168c5-f04c-44ff-8c17-d0ac0abfc8ce reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:00:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4e069fd4-f6dd-4552-92fc-2519b81e510b kind: Event lastTimestamp: "2026-07-16T22:01:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:00:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:17Z" name: isvc-sklearn-graph-1-predictor.18c2e420cfe8a5fd namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: 7233c901-76c6-4fac-9b77-24dcff1d0230 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:00:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4e069fd4-f6dd-4552-92fc-2519b81e510b kind: Event lastTimestamp: "2026-07-16T22:01:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:00:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:17Z" name: isvc-sklearn-graph-1-predictor.18c2e420d05d2747 namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: 3437c758-d696-4c7c-8faf-a2694a7b61a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:01:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: 4e069fd4-f6dd-4552-92fc-2519b81e510b kind: Event lastTimestamp: "2026-07-16T22:02:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:17Z" name: isvc-sklearn-graph-1-predictor.18c2e42ecb7b76ae namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 5253a08a-c3e1-476b-a0a3-b98097236c79 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:01:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: 4e069fd4-f6dd-4552-92fc-2519b81e510b kind: Event lastTimestamp: "2026-07-16T22:02:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:17Z" name: isvc-sklearn-graph-1-predictor.18c2e42ecbec943f namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 533290cd-4c20-4636-9268-c441e2661aae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-16T22:00:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: 89e32963-db55-4fc9-82da-a02680adfbde kind: Event lastTimestamp: "2026-07-16T22:02:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:02:05Z" name: isvc-sklearn-graph-1.18c2e41d5154c8db namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: 6b212113-5b64-44b3-9523-c76742eec9b6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 89e32963-db55-4fc9-82da-a02680adfbde kind: Event lastTimestamp: "2026-07-16T22:02:05Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-16T22:02:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:02:05Z" name: isvc-sklearn-graph-1.18c2e436a761cd09 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 373f7a30-13e5-44da-86aa-6b69caec4a2e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:00:17.644302Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14748" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 to ip-10-0-128-250.ec2.internal metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:00:17Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e41d7a028c1c namespace: kserve-ci-e2e-test resourceVersion: "14752" uid: 0d96b087-dc0b-4bcc-ba8a-97d4266cc26c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:00:18Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:00:18Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e41d9412a532 namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 17134c62-8e26-41ac-badf-cfa75d92880e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:00:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5d63c390841fcd6358d7b4068789845000a8eff3dfe19f0f9b697dde4d31930a" already present on machine metadata: creationTimestamp: "2026-07-16T22:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:18Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e41d95a79de7 namespace: kserve-ci-e2e-test resourceVersion: "14793" uid: e8b60504-229f-47a2-bdec-d48a357b2a44 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:00:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T22:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:18Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e41d98cbee27 namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 0f75f9d8-b865-400c-91b1-202688357a67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:00:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T22:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:18Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e41d994182e4 namespace: kserve-ci-e2e-test resourceVersion: "14797" uid: fcb8f35c-cbc5-4268-9e61-c3dd88297bbc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:00:21Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1745" metadata: creationTimestamp: "2026-07-16T22:00:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:21Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e41e76ef26a3 namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 07b01fd1-ec73-481c-92db-cc3c9d50d901 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:00:49Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1745" in 27.568s (27.568s including waiting). Image size: 1555872170 bytes.' metadata: creationTimestamp: "2026-07-16T22:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:49Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e424e22ab07f namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 3a0ed29a-0c78-428a-8329-7330473142a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:00:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:49Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e424e5ad4b79 namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 08b82ea3-61c9-4176-b8dc-bcb46afa3661 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:00:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:49Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e424e639beb9 namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: 4c3edef2-9b5d-4b96-9c67-51876cc771b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:00:49Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-16T22:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:49Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e424e647496e namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: 0a68d5b6-27bd-4ac8-b13a-8ba694f4fa49 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:00:52Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.935s (2.935s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-16T22:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:52Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e425953d4b52 namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: 83cd5a4d-90c0-4b9c-af49-7cf267eeb771 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:00:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:52Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e425986916ec namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 5bfeb80d-d2a0-4a45-b2e6-731510d1c579 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:00:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:52Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e42598ea84b7 namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 4066e823-079e-4b51-ac3d-ce1b4d76dfa2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-16T22:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:02:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:38Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e425f80b0b43 namespace: kserve-ci-e2e-test resourceVersion: "16428" uid: 1189ba32-c888-47ab-a1d0-8af66b3fe6a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:37Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e43e070da7a2 namespace: kserve-ci-e2e-test resourceVersion: "16389" uid: 6249a230-a1b1-4142-953f-3026480e2211 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:37Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e43e070eeb90 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: 8a8d24c7-a76f-4d06-89fc-206a64683e8e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 3e2b6cce-e7ef-4b78-a032-5c2657ba8592 kind: Event lastTimestamp: "2026-07-16T22:02:40Z" message: 'Readiness probe failed: Get "https://10.132.0.19:8643/healthz": dial tcp 10.132.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:40Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6.18c2e43ea657eae1 namespace: kserve-ci-e2e-test resourceVersion: "16450" uid: 65ac43a1-35a9-49bc-96f7-757d2703d7c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-779dbb5fd4 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 436303d3-5af8-4197-b82d-910baa4e7459 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-779dbb5fd4-kvfc6' metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:00:17Z" name: isvc-sklearn-graph-2-predictor-779dbb5fd4.18c2e41d795ad122 namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: 6d1424f2-cb93-4f7b-9261-2df433f26238 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: d085620e-6530-49b9-87dc-2b643d317675 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-779dbb5fd4 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:00:17Z" name: isvc-sklearn-graph-2-predictor.18c2e41d7601c695 namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: cd26d15c-d705-4b97-b000-1fd2825b4610 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:00:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: 0dc84a3d-bbf0-4678-a44a-c02f4285476d kind: Event lastTimestamp: "2026-07-16T22:01:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:00:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:17Z" name: isvc-sklearn-graph-2-predictor.18c2e420f4c8470a namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: 02f441d5-2eb2-40e9-a3c0-213a9d138660 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:00:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: 0dc84a3d-bbf0-4678-a44a-c02f4285476d kind: Event lastTimestamp: "2026-07-16T22:01:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:00:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:17Z" name: isvc-sklearn-graph-2-predictor.18c2e420f5185e6f namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 7163cb54-e7eb-47ac-b397-6d9a828cb6d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:01:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 0dc84a3d-bbf0-4678-a44a-c02f4285476d kind: Event lastTimestamp: "2026-07-16T22:02:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:32Z" name: isvc-sklearn-graph-2-predictor.18c2e42ef079a3db namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: 00f9c13b-6fea-4aa7-9c44-4ca1b719c1ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:01:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 0dc84a3d-bbf0-4678-a44a-c02f4285476d kind: Event lastTimestamp: "2026-07-16T22:02:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:32Z" name: isvc-sklearn-graph-2-predictor.18c2e42ef0f304fb namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: 85854d68-d660-4e50-8ee4-cccd08044bf7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14717" uid: 46765ff8-cb2d-4273-9639-fb380abea066 kind: Event lastTimestamp: "2026-07-16T22:02:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:02:08Z" name: isvc-sklearn-graph-2.18c2e41d7649a476 namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: ed47bc4a-5da8-4dea-8777-e5983cb16bbd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 46765ff8-cb2d-4273-9639-fb380abea066 kind: Event lastTimestamp: "2026-07-16T22:02:08Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-16T22:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:02:08Z" name: isvc-sklearn-graph-2.18c2e4375a0eb9d8 namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 9f69e8e3-9eb5-4291-8894-43bf11603118 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:00:17.517022Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-ptfhh to ip-10-0-128-250.ec2.internal metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:00:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e41d726c6f93 namespace: kserve-ci-e2e-test resourceVersion: "14723" uid: ee441b80-bac3-4e7b-9460-f237e6d7b54c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14728" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:00:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e41d8c3d3aa4 namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: f5443b58-338d-4661-9043-50a161fdca3e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5d63c390841fcd6358d7b4068789845000a8eff3dfe19f0f9b697dde4d31930a" already present on machine metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e41d8e2fc9a8 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: 1c9a67a9-e6b7-4234-84af-35242cb36f15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:00:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-16T22:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e41d91d0be84 namespace: kserve-ci-e2e-test resourceVersion: "14788" uid: 63658fa8-5f60-47f0-821d-c569b0a33c23 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:00:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-16T22:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e41d9253ad58 namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 099f9645-55f4-49b3-9a03-e03d7ef4a888 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:00:21Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-16T22:00:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e41e76d9b99f namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: c62e9f56-8355-42c7-87f4-fb311fec3a3e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:00:49Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 27.68s (27.68s including waiting). Image size: 1306969338 bytes.' metadata: creationTimestamp: "2026-07-16T22:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e424e8c1a477 namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: c7ff11df-314a-4ce6-b688-fd4d0db9c4f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:00:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e424ebe930d2 namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: 2e0eb47f-7a44-48fd-a19c-12f761dce300 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:00:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e424ec5783f2 namespace: kserve-ci-e2e-test resourceVersion: "15103" uid: 47813dea-f429-4c52-866f-008aa3c3182b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:00:49Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-16T22:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e424ec629724 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 2eebb56c-efbb-4e50-9dbd-63ed11789d6f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:00:52Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.843s (2.843s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-16T22:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e42595dfac26 namespace: kserve-ci-e2e-test resourceVersion: "15129" uid: fd37414a-ffe5-4f0d-bd26-f91e838edf81 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:00:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e42598dde3ca namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: a99edb1a-df94-4eb0-8620-5b10b6062dec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:00:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e425994d62f8 namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: eb8fe4f5-0c22-4aeb-9b46-bf6c386442d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-16T22:00:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:02:39Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:00:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e425bc10e3db namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: 68ef3d2d-338d-4f48-948f-d6bb4bf1a7d3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e43dfe97abc6 namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: f49f76a5-7c50-4222-a30d-28a880f3bcfd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e43dfe98660d namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: 561c5c3a-fe20-4673-8a57-265f20060055 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 406a6ace-a702-4e4c-8f45-66442558ff36 kind: Event lastTimestamp: "2026-07-16T22:02:39Z" message: 'Readiness probe failed: Get "https://10.132.0.18:8643/healthz": dial tcp 10.132.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-ptfhh.18c2e43e6a8812aa namespace: kserve-ci-e2e-test resourceVersion: "16440" uid: 88b9182e-6c99-49cf-8122-f761b0f8dbd7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-250.ec2.internal source: component: kubelet host: ip-10-0-128-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: 755756f2-83b6-46b9-8acb-c66b87934c1f kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-ptfhh' metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:00:17Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c2e41d71ff76cf namespace: kserve-ci-e2e-test resourceVersion: "14719" uid: a0fa6017-875e-437c-b719-cb6044dba3f9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 684e2bde-ce3b-4326-bd80-f673328aa8e5 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:00:17Z" name: isvc-xgboost-graph-predictor.18c2e41d6e2a0804 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: f5845612-11f8-4174-8129-edff3ede49af reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:00:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: 4c83fb91-5071-4e32-bc9e-baab36cda18f kind: Event lastTimestamp: "2026-07-16T22:01:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:00:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:17Z" name: isvc-xgboost-graph-predictor.18c2e420ec972054 namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: cce870a4-6b60-46a3-aac0-c8fff64b3f12 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:00:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: 4c83fb91-5071-4e32-bc9e-baab36cda18f kind: Event lastTimestamp: "2026-07-16T22:01:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:00:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:17Z" name: isvc-xgboost-graph-predictor.18c2e420ed0a4833 namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: 8f5604fc-e0fa-4a51-ad43-16b8d70c3022 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:01:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: 4c83fb91-5071-4e32-bc9e-baab36cda18f kind: Event lastTimestamp: "2026-07-16T22:02:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:02Z" name: isvc-xgboost-graph-predictor.18c2e42ee8474c63 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 23b346d7-8636-4b05-b7fc-a047c13a7e92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:01:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: 4c83fb91-5071-4e32-bc9e-baab36cda18f kind: Event lastTimestamp: "2026-07-16T22:02:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:01:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:02Z" name: isvc-xgboost-graph-predictor.18c2e42ee8b65100 namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: e4d84765-f988-4d4c-9850-ee6c47997a13 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: 2eab176d-5e41-45e3-bec4-694be9449914 kind: Event lastTimestamp: "2026-07-16T22:01:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:01:49Z" name: isvc-xgboost-graph.18c2e41d6e294bd0 namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: 80929c6d-3797-4975-ac38-7023a40ec195 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 2eab176d-5e41-45e3-bec4-694be9449914 kind: Event lastTimestamp: "2026-07-16T22:01:49Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-16T22:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:01:49Z" name: isvc-xgboost-graph.18c2e432cc10eb00 namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: 775e19e3-0bea-4d5c-b34c-9a41786735c4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:02:26.863906Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5b578b95c5-5vjwk namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 1ea7e54b-fa93-44fa-86fd-e3bd5c447a2b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-5b578b95c5-5vjwk to ip-10-0-131-231.ec2.internal metadata: creationTimestamp: "2026-07-16T22:02:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:02:26Z" name: model-chainer-5b578b95c5-5vjwk.18c2e43b90193ab8 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 9ea4feaf-2792-4789-8343-57cdea446f4c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:27Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5b578b95c5-5vjwk namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: 1ea7e54b-fa93-44fa-86fd-e3bd5c447a2b kind: Event lastTimestamp: "2026-07-16T22:02:27Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:02:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:02:27Z" name: model-chainer-5b578b95c5-5vjwk.18c2e43baa0c9844 namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: ebebcdb7-7810-4dbd-ac2e-8931148de7d5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5b578b95c5-5vjwk namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 1ea7e54b-fa93-44fa-86fd-e3bd5c447a2b kind: Event lastTimestamp: "2026-07-16T22:02:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7e35538adc84c05d9420d87001e9355aeddffd6326bb391010f8e6c1a3c85203" already present on machine metadata: creationTimestamp: "2026-07-16T22:02:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:27Z" name: model-chainer-5b578b95c5-5vjwk.18c2e43bab5bf933 namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 4ff96c32-7802-4b70-afb1-e17fbbf2c12f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5b578b95c5-5vjwk namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 1ea7e54b-fa93-44fa-86fd-e3bd5c447a2b kind: Event lastTimestamp: "2026-07-16T22:02:27Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-16T22:02:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:27Z" name: model-chainer-5b578b95c5-5vjwk.18c2e43bae34caab namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: bcc8e928-41b6-4ab9-b109-dadbfea95a8a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5b578b95c5-5vjwk namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 1ea7e54b-fa93-44fa-86fd-e3bd5c447a2b kind: Event lastTimestamp: "2026-07-16T22:02:27Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-16T22:02:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:27Z" name: model-chainer-5b578b95c5-5vjwk.18c2e43bae9f5da8 namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: e9a62ab1-bc17-4abc-90c8-93a2458020af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5b578b95c5-5vjwk namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 1ea7e54b-fa93-44fa-86fd-e3bd5c447a2b kind: Event lastTimestamp: "2026-07-16T22:02:36Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-16T22:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:36Z" name: model-chainer-5b578b95c5-5vjwk.18c2e43deaba74f2 namespace: kserve-ci-e2e-test resourceVersion: "16285" uid: 2ef0f1e6-ee30-4842-b46e-81b248ddc0c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:02:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5b578b95c5-5vjwk namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 1ea7e54b-fa93-44fa-86fd-e3bd5c447a2b kind: Event lastTimestamp: "2026-07-16T22:03:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T22:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:03:04Z" name: model-chainer-5b578b95c5-5vjwk.18c2e43e6f4562fd namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: 9dc031e9-c7c2-4e3c-b7bd-392d9aa229af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-5b578b95c5 namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: 0aba3707-061a-4977-ac1b-ac814ee32a14 kind: Event lastTimestamp: "2026-07-16T22:02:26Z" message: 'Created pod: model-chainer-5b578b95c5-5vjwk' metadata: creationTimestamp: "2026-07-16T22:02:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:26Z" name: model-chainer-5b578b95c5.18c2e43b8f88a0e4 namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: d78cd6e3-9368-4b47-b202-a116abe5f694 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: 654a8196-6fae-4e8d-bfbc-65021d1972c8 kind: Event lastTimestamp: "2026-07-16T22:02:26Z" message: Scaled up replica set model-chainer-5b578b95c5 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:02:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:26Z" name: model-chainer.18c2e43b8eb2798f namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 7aee8216-550c-433c-9ab8-d477598afa18 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16169" uid: 03b9e952-16e2-49ac-ac38-064a1c95816c kind: Event lastTimestamp: "2026-07-16T22:02:26Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-16T22:02:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:02:26Z" name: model-chainer.18c2e43b94789e3a namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: f0a84498-f44d-4219-8a30-0c35cf1b75ff reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:13:02.136368Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-67446-6fbc776749-h7b4k namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: 329a3eaa-8926-4198-a7d2-142cac7fb99e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-67446-6fbc776749-h7b4k to ip-10-0-131-231.ec2.internal metadata: creationTimestamp: "2026-07-16T22:13:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:13:02Z" name: sequence-graph-67446-6fbc776749-h7b4k.18c2e4cf7949c3bd namespace: kserve-ci-e2e-test resourceVersion: "21517" uid: d2c8a23a-89fa-4fb8-bdd6-91e03ea94473 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-67446-6fbc776749-h7b4k namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: 329a3eaa-8926-4198-a7d2-142cac7fb99e kind: Event lastTimestamp: "2026-07-16T22:13:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-67446-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:02Z" name: sequence-graph-67446-6fbc776749-h7b4k.18c2e4cf815b6f20 namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: 3bfca3a0-b53f-4f64-82de-54d96364459a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:03Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-67446-6fbc776749-h7b4k namespace: kserve-ci-e2e-test resourceVersion: "21521" uid: 329a3eaa-8926-4198-a7d2-142cac7fb99e kind: Event lastTimestamp: "2026-07-16T22:13:03Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:13:03Z" name: sequence-graph-67446-6fbc776749-h7b4k.18c2e4cfb6b3460d namespace: kserve-ci-e2e-test resourceVersion: "21539" uid: ab2e9ea1-a11b-42b7-9d6d-e27197566e29 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-67446} kind: Pod name: sequence-graph-67446-6fbc776749-h7b4k namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: 329a3eaa-8926-4198-a7d2-142cac7fb99e kind: Event lastTimestamp: "2026-07-16T22:13:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7e35538adc84c05d9420d87001e9355aeddffd6326bb391010f8e6c1a3c85203" already present on machine metadata: creationTimestamp: "2026-07-16T22:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:03Z" name: sequence-graph-67446-6fbc776749-h7b4k.18c2e4cfb801875b namespace: kserve-ci-e2e-test resourceVersion: "21541" uid: 18689ec0-e8ef-4884-9f85-3b14d433582a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-67446} kind: Pod name: sequence-graph-67446-6fbc776749-h7b4k namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: 329a3eaa-8926-4198-a7d2-142cac7fb99e kind: Event lastTimestamp: "2026-07-16T22:13:03Z" message: 'Created container: sequence-graph-67446' metadata: creationTimestamp: "2026-07-16T22:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:03Z" name: sequence-graph-67446-6fbc776749-h7b4k.18c2e4cfbabf1ab5 namespace: kserve-ci-e2e-test resourceVersion: "21542" uid: a10c174b-8d93-450c-a83a-463ea8a4813b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-67446} kind: Pod name: sequence-graph-67446-6fbc776749-h7b4k namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: 329a3eaa-8926-4198-a7d2-142cac7fb99e kind: Event lastTimestamp: "2026-07-16T22:13:03Z" message: Started container sequence-graph-67446 metadata: creationTimestamp: "2026-07-16T22:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:03Z" name: sequence-graph-67446-6fbc776749-h7b4k.18c2e4cfbb35a7ee namespace: kserve-ci-e2e-test resourceVersion: "21543" uid: 411f896d-2d2d-4a8b-95f5-263744ef3458 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-67446} kind: Pod name: sequence-graph-67446-6fbc776749-h7b4k namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: 329a3eaa-8926-4198-a7d2-142cac7fb99e kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: Stopping container sequence-graph-67446 metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:12Z" name: sequence-graph-67446-6fbc776749-h7b4k.18c2e4d1d3c65e7b namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: d2f2da54-ccb6-4d6c-b039-f1dc3c935bd4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:13:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-67446} kind: Pod name: sequence-graph-67446-6fbc776749-h7b4k namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: 329a3eaa-8926-4198-a7d2-142cac7fb99e kind: Event lastTimestamp: "2026-07-16T22:13:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T22:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:39Z" name: sequence-graph-67446-6fbc776749-h7b4k.18c2e4d25b99e004 namespace: kserve-ci-e2e-test resourceVersion: "22018" uid: 2b21fec8-179d-46d4-a2e7-71c820de44dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-67446-6fbc776749 namespace: kserve-ci-e2e-test resourceVersion: "21509" uid: af4845d7-ebf0-41c7-b515-f2207fb52cdf kind: Event lastTimestamp: "2026-07-16T22:13:02Z" message: 'Created pod: sequence-graph-67446-6fbc776749-h7b4k' metadata: creationTimestamp: "2026-07-16T22:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:02Z" name: sequence-graph-67446-6fbc776749.18c2e4cf78d66de6 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: cef88cf7-10bf-499c-9990-4ddbcf1346d6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-67446 namespace: kserve-ci-e2e-test resourceVersion: "21507" uid: b3ca65d4-ae3d-4297-91fb-8599fefb07e5 kind: Event lastTimestamp: "2026-07-16T22:13:02Z" message: Scaled up replica set sequence-graph-67446-6fbc776749 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:02Z" name: sequence-graph-67446.18c2e4cf77fd3b17 namespace: kserve-ci-e2e-test resourceVersion: "21510" uid: 2bccc39f-d033-413e-98c6-54800379a877 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-67446 namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: 16b5c4cb-75bd-4f48-ab85-a4817d107b85 kind: Event lastTimestamp: "2026-07-16T22:13:02Z" message: InferenceGraph [sequence-graph-67446] is Ready metadata: creationTimestamp: "2026-07-16T22:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:13:02Z" name: sequence-graph-67446.18c2e4cf7df2158a namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: d760e794-550e-480e-a35c-8b78705d5b54 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:03:37.209439Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-77f35-5678976b5d-lxn82 namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: da114ca6-b44d-4a4b-8415-3e17a65a6995 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-77f35-5678976b5d-lxn82 to ip-10-0-131-231.ec2.internal metadata: creationTimestamp: "2026-07-16T22:03:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:03:37Z" name: sequence-graph-77f35-5678976b5d-lxn82.18c2e44bf104eec2 namespace: kserve-ci-e2e-test resourceVersion: "16993" uid: 293d8661-8ab0-4c83-93c7-060a1aacd65b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:37Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-77f35-5678976b5d-lxn82 namespace: kserve-ci-e2e-test resourceVersion: "16997" uid: da114ca6-b44d-4a4b-8415-3e17a65a6995 kind: Event lastTimestamp: "2026-07-16T22:03:37Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:03:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:03:37Z" name: sequence-graph-77f35-5678976b5d-lxn82.18c2e44c0a401180 namespace: kserve-ci-e2e-test resourceVersion: "17012" uid: a07d29c1-3742-49db-8de0-4119e18af6f3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-77f35} kind: Pod name: sequence-graph-77f35-5678976b5d-lxn82 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: da114ca6-b44d-4a4b-8415-3e17a65a6995 kind: Event lastTimestamp: "2026-07-16T22:03:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7e35538adc84c05d9420d87001e9355aeddffd6326bb391010f8e6c1a3c85203" already present on machine metadata: creationTimestamp: "2026-07-16T22:03:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:03:37Z" name: sequence-graph-77f35-5678976b5d-lxn82.18c2e44c0b9bdee2 namespace: kserve-ci-e2e-test resourceVersion: "17014" uid: 3b474f0c-0bea-445e-8f41-9b187e49313d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-77f35} kind: Pod name: sequence-graph-77f35-5678976b5d-lxn82 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: da114ca6-b44d-4a4b-8415-3e17a65a6995 kind: Event lastTimestamp: "2026-07-16T22:03:37Z" message: 'Created container: sequence-graph-77f35' metadata: creationTimestamp: "2026-07-16T22:03:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:03:37Z" name: sequence-graph-77f35-5678976b5d-lxn82.18c2e44c0e4b7451 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: 60ba6a30-39c1-4024-afe4-6ef74b2c8dbc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-77f35} kind: Pod name: sequence-graph-77f35-5678976b5d-lxn82 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: da114ca6-b44d-4a4b-8415-3e17a65a6995 kind: Event lastTimestamp: "2026-07-16T22:03:37Z" message: Started container sequence-graph-77f35 metadata: creationTimestamp: "2026-07-16T22:03:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:03:37Z" name: sequence-graph-77f35-5678976b5d-lxn82.18c2e44c0eb3f456 namespace: kserve-ci-e2e-test resourceVersion: "17016" uid: 6c3c1735-2d50-40d1-bc88-279352159d82 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-77f35} kind: Pod name: sequence-graph-77f35-5678976b5d-lxn82 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: da114ca6-b44d-4a4b-8415-3e17a65a6995 kind: Event lastTimestamp: "2026-07-16T22:11:51Z" message: Stopping container sequence-graph-77f35 metadata: creationTimestamp: "2026-07-16T22:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:51Z" name: sequence-graph-77f35-5678976b5d-lxn82.18c2e4bf1f0d72e7 namespace: kserve-ci-e2e-test resourceVersion: "20590" uid: b496c1a0-7dce-43ae-9939-80aaec148372 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-77f35} kind: Pod name: sequence-graph-77f35-5678976b5d-lxn82 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: da114ca6-b44d-4a4b-8415-3e17a65a6995 kind: Event lastTimestamp: "2026-07-16T22:12:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T22:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:19Z" name: sequence-graph-77f35-5678976b5d-lxn82.18c2e4bfb7513545 namespace: kserve-ci-e2e-test resourceVersion: "20960" uid: 76575e08-1304-4012-ba82-08fcfde4592f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-77f35-5678976b5d namespace: kserve-ci-e2e-test resourceVersion: "16985" uid: 668faa9c-718e-4395-926c-e87b83628a84 kind: Event lastTimestamp: "2026-07-16T22:03:37Z" message: 'Created pod: sequence-graph-77f35-5678976b5d-lxn82' metadata: creationTimestamp: "2026-07-16T22:03:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:37Z" name: sequence-graph-77f35-5678976b5d.18c2e44bf083d0ba namespace: kserve-ci-e2e-test resourceVersion: "16989" uid: fd2b1da7-b3cc-4964-89bd-0279bf18af13 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-77f35 namespace: kserve-ci-e2e-test resourceVersion: "16983" uid: 419d571f-0b34-4ae8-8d5d-0a6b2c176dd3 kind: Event lastTimestamp: "2026-07-16T22:03:37Z" message: Scaled up replica set sequence-graph-77f35-5678976b5d from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:03:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:37Z" name: sequence-graph-77f35.18c2e44bef97894a namespace: kserve-ci-e2e-test resourceVersion: "16986" uid: d448b81b-4e5e-43a2-8676-07fc66f6406d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-77f35 namespace: kserve-ci-e2e-test resourceVersion: "17003" uid: 137af440-9ce8-4200-9c4c-f09cc60338d3 kind: Event lastTimestamp: "2026-07-16T22:03:37Z" message: InferenceGraph [sequence-graph-77f35] is Ready metadata: creationTimestamp: "2026-07-16T22:03:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:03:37Z" name: sequence-graph-77f35.18c2e44bf7198ac0 namespace: kserve-ci-e2e-test resourceVersion: "17005" uid: 002be267-d34f-4c66-9e09-10294ae0b535 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-77f35 namespace: kserve-ci-e2e-test resourceVersion: "16984" uid: fc73aa90-6c6e-438f-beca-a634a477a6c4 kind: Event lastTimestamp: "2026-07-16T22:03:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:52Z" name: sequence-graph-77f35.18c2e44f6e358e6e namespace: kserve-ci-e2e-test resourceVersion: "17131" uid: 745be6c1-268b-44b2-b50e-c3631ba1c183 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-77f35 namespace: kserve-ci-e2e-test resourceVersion: "16984" uid: fc73aa90-6c6e-438f-beca-a634a477a6c4 kind: Event lastTimestamp: "2026-07-16T22:03:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:52Z" name: sequence-graph-77f35.18c2e44f6e9c8cef namespace: kserve-ci-e2e-test resourceVersion: "17133" uid: e7fc392f-89ec-4f22-9c79-2dd370c8acf0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:04:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-77f35 namespace: kserve-ci-e2e-test resourceVersion: "17132" uid: fc73aa90-6c6e-438f-beca-a634a477a6c4 kind: Event lastTimestamp: "2026-07-16T22:04:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:04:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:04:07Z" name: sequence-graph-77f35.18c2e452ed637488 namespace: kserve-ci-e2e-test resourceVersion: "17241" uid: 8c5a8c29-0c35-44b0-8d01-7a7f1214371b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:04:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-77f35 namespace: kserve-ci-e2e-test resourceVersion: "17132" uid: fc73aa90-6c6e-438f-beca-a634a477a6c4 kind: Event lastTimestamp: "2026-07-16T22:04:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:04:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:04:07Z" name: sequence-graph-77f35.18c2e452eddbbece namespace: kserve-ci-e2e-test resourceVersion: "17243" uid: f76ec295-5372-4a10-b947-36c10268a99e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:14:22.512827Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a3a91-54fb849f7b-5fgx5 namespace: kserve-ci-e2e-test resourceVersion: "22413" uid: afcdd5f1-6428-40cd-bd8c-851217ba3a6b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a3a91-54fb849f7b-5fgx5 to ip-10-0-131-231.ec2.internal metadata: creationTimestamp: "2026-07-16T22:14:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:14:22Z" name: sequence-graph-a3a91-54fb849f7b-5fgx5.18c2e4e23019315c namespace: kserve-ci-e2e-test resourceVersion: "22418" uid: 46e07c9a-701d-40ef-acef-86d219b4374e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:22Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a3a91-54fb849f7b-5fgx5 namespace: kserve-ci-e2e-test resourceVersion: "22415" uid: afcdd5f1-6428-40cd-bd8c-851217ba3a6b kind: Event lastTimestamp: "2026-07-16T22:14:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-a3a91-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:14:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:14:22Z" name: sequence-graph-a3a91-54fb849f7b-5fgx5.18c2e4e23c0aed91 namespace: kserve-ci-e2e-test resourceVersion: "22432" uid: d12b8101-2bd5-4349-8fcb-98826f733c8e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:23Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a3a91-54fb849f7b-5fgx5 namespace: kserve-ci-e2e-test resourceVersion: "22421" uid: afcdd5f1-6428-40cd-bd8c-851217ba3a6b kind: Event lastTimestamp: "2026-07-16T22:14:23Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:14:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:14:23Z" name: sequence-graph-a3a91-54fb849f7b-5fgx5.18c2e4e26d8ef43c namespace: kserve-ci-e2e-test resourceVersion: "22439" uid: b95ab3af-dd8d-46c6-bf90-2e14d9eb1e93 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3a91} kind: Pod name: sequence-graph-a3a91-54fb849f7b-5fgx5 namespace: kserve-ci-e2e-test resourceVersion: "22415" uid: afcdd5f1-6428-40cd-bd8c-851217ba3a6b kind: Event lastTimestamp: "2026-07-16T22:14:23Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7e35538adc84c05d9420d87001e9355aeddffd6326bb391010f8e6c1a3c85203" already present on machine metadata: creationTimestamp: "2026-07-16T22:14:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:14:23Z" name: sequence-graph-a3a91-54fb849f7b-5fgx5.18c2e4e26ed22e1a namespace: kserve-ci-e2e-test resourceVersion: "22441" uid: 81975efa-08b3-4c81-9916-3388817ffe79 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3a91} kind: Pod name: sequence-graph-a3a91-54fb849f7b-5fgx5 namespace: kserve-ci-e2e-test resourceVersion: "22415" uid: afcdd5f1-6428-40cd-bd8c-851217ba3a6b kind: Event lastTimestamp: "2026-07-16T22:14:23Z" message: 'Created container: sequence-graph-a3a91' metadata: creationTimestamp: "2026-07-16T22:14:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:14:23Z" name: sequence-graph-a3a91-54fb849f7b-5fgx5.18c2e4e271f11f1c namespace: kserve-ci-e2e-test resourceVersion: "22442" uid: 21f70495-e088-4f27-9611-630a3340e38d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3a91} kind: Pod name: sequence-graph-a3a91-54fb849f7b-5fgx5 namespace: kserve-ci-e2e-test resourceVersion: "22415" uid: afcdd5f1-6428-40cd-bd8c-851217ba3a6b kind: Event lastTimestamp: "2026-07-16T22:14:23Z" message: Started container sequence-graph-a3a91 metadata: creationTimestamp: "2026-07-16T22:14:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:14:23Z" name: sequence-graph-a3a91-54fb849f7b-5fgx5.18c2e4e27266788a namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: 9f4a57b8-5ea1-4350-ad0d-3dc5732b13d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3a91} kind: Pod name: sequence-graph-a3a91-54fb849f7b-5fgx5 namespace: kserve-ci-e2e-test resourceVersion: "22415" uid: afcdd5f1-6428-40cd-bd8c-851217ba3a6b kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: Stopping container sequence-graph-a3a91 metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:37Z" name: sequence-graph-a3a91-54fb849f7b-5fgx5.18c2e5555e735769 namespace: kserve-ci-e2e-test resourceVersion: "26305" uid: 8ec1c446-c5f2-4f5f-b7cc-6672fb4f3499 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:22:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a3a91} kind: Pod name: sequence-graph-a3a91-54fb849f7b-5fgx5 namespace: kserve-ci-e2e-test resourceVersion: "22415" uid: afcdd5f1-6428-40cd-bd8c-851217ba3a6b kind: Event lastTimestamp: "2026-07-16T22:23:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T22:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:04Z" name: sequence-graph-a3a91-54fb849f7b-5fgx5.18c2e555fa19c7f5 namespace: kserve-ci-e2e-test resourceVersion: "26679" uid: 8a567436-bed9-4778-b9c3-094d4aa22640 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a3a91-54fb849f7b namespace: kserve-ci-e2e-test resourceVersion: "22410" uid: af449e98-7c01-4207-8b20-6499d01929a8 kind: Event lastTimestamp: "2026-07-16T22:14:22Z" message: 'Created pod: sequence-graph-a3a91-54fb849f7b-5fgx5' metadata: creationTimestamp: "2026-07-16T22:14:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:22Z" name: sequence-graph-a3a91-54fb849f7b.18c2e4e22fa8990c namespace: kserve-ci-e2e-test resourceVersion: "22414" uid: 53098d09-fe41-47df-9295-4086865c6d43 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "22408" uid: 52afe28a-0047-4a5e-ac66-afd9a1e513b9 kind: Event lastTimestamp: "2026-07-16T22:14:22Z" message: Scaled up replica set sequence-graph-a3a91-54fb849f7b from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:14:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:22Z" name: sequence-graph-a3a91.18c2e4e22ec3683c namespace: kserve-ci-e2e-test resourceVersion: "22412" uid: f6f9c0ce-3e16-488f-aa5b-b4f26f5a39c9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "22424" uid: 52c15128-1225-4c06-8bc5-81b9d91e0f96 kind: Event lastTimestamp: "2026-07-16T22:14:22Z" message: InferenceGraph [sequence-graph-a3a91] is Ready metadata: creationTimestamp: "2026-07-16T22:14:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:14:22Z" name: sequence-graph-a3a91.18c2e4e23484f35b namespace: kserve-ci-e2e-test resourceVersion: "22427" uid: 4cc8a632-c060-4f6c-9a71-ecf1998bd8d4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "22409" uid: 68108c9a-3964-4b3a-8981-7d9e1dbb2d46 kind: Event lastTimestamp: "2026-07-16T22:14:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:14:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:37Z" name: sequence-graph-a3a91.18c2e4e5adb0df65 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 53e61975-64c9-4410-aa8b-c6f52c64817d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "22409" uid: 68108c9a-3964-4b3a-8981-7d9e1dbb2d46 kind: Event lastTimestamp: "2026-07-16T22:14:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:14:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:37Z" name: sequence-graph-a3a91.18c2e4e5ae55a034 namespace: kserve-ci-e2e-test resourceVersion: "22560" uid: 5dd5eb5c-2646-43dc-a12d-c5d1725e3414 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "22558" uid: 68108c9a-3964-4b3a-8981-7d9e1dbb2d46 kind: Event lastTimestamp: "2026-07-16T22:14:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:52Z" name: sequence-graph-a3a91.18c2e4e92d236597 namespace: kserve-ci-e2e-test resourceVersion: "22657" uid: d5960232-a730-46a1-b63f-0f683d707a2a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "22558" uid: 68108c9a-3964-4b3a-8981-7d9e1dbb2d46 kind: Event lastTimestamp: "2026-07-16T22:14:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:52Z" name: sequence-graph-a3a91.18c2e4e92d8f25a7 namespace: kserve-ci-e2e-test resourceVersion: "22659" uid: 59009254-0d9b-4a53-b136-557a99935e2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:24:21.966328Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-76a97-8669877574-rbn4w namespace: kserve-ci-e2e-test resourceVersion: "27559" uid: e2e1c182-96b5-478f-a3a5-b5bef18bc6ed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-76a97-8669877574-rbn4w to ip-10-0-131-231.ec2.internal metadata: creationTimestamp: "2026-07-16T22:24:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:24:21Z" name: splitter-graph-76a97-8669877574-rbn4w.18c2e56dc24fafc8 namespace: kserve-ci-e2e-test resourceVersion: "27566" uid: 9e0b10b8-fdd2-48e4-8aa8-bac62092c7b7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:22Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-76a97-8669877574-rbn4w namespace: kserve-ci-e2e-test resourceVersion: "27560" uid: e2e1c182-96b5-478f-a3a5-b5bef18bc6ed kind: Event lastTimestamp: "2026-07-16T22:24:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-76a97-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:24:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:24:22Z" name: splitter-graph-76a97-8669877574-rbn4w.18c2e56dd065a116 namespace: kserve-ci-e2e-test resourceVersion: "27577" uid: 3acafdd5-d897-4132-aff7-2f9db61a736f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:22Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-76a97-8669877574-rbn4w namespace: kserve-ci-e2e-test resourceVersion: "27568" uid: e2e1c182-96b5-478f-a3a5-b5bef18bc6ed kind: Event lastTimestamp: "2026-07-16T22:24:22Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:24:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:24:22Z" name: splitter-graph-76a97-8669877574-rbn4w.18c2e56dff968abe namespace: kserve-ci-e2e-test resourceVersion: "27591" uid: dc43a9c4-13f1-4f7c-b179-80def5352f4b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-76a97} kind: Pod name: splitter-graph-76a97-8669877574-rbn4w namespace: kserve-ci-e2e-test resourceVersion: "27560" uid: e2e1c182-96b5-478f-a3a5-b5bef18bc6ed kind: Event lastTimestamp: "2026-07-16T22:24:23Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7e35538adc84c05d9420d87001e9355aeddffd6326bb391010f8e6c1a3c85203" already present on machine metadata: creationTimestamp: "2026-07-16T22:24:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:24:23Z" name: splitter-graph-76a97-8669877574-rbn4w.18c2e56e0103855c namespace: kserve-ci-e2e-test resourceVersion: "27594" uid: 40a3fd77-80cf-431e-b607-c4f78b80f1c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-76a97} kind: Pod name: splitter-graph-76a97-8669877574-rbn4w namespace: kserve-ci-e2e-test resourceVersion: "27560" uid: e2e1c182-96b5-478f-a3a5-b5bef18bc6ed kind: Event lastTimestamp: "2026-07-16T22:24:23Z" message: 'Created container: splitter-graph-76a97' metadata: creationTimestamp: "2026-07-16T22:24:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:24:23Z" name: splitter-graph-76a97-8669877574-rbn4w.18c2e56e04ec771f namespace: kserve-ci-e2e-test resourceVersion: "27595" uid: f91940e8-5c45-4c49-83e6-1da142672d70 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-76a97} kind: Pod name: splitter-graph-76a97-8669877574-rbn4w namespace: kserve-ci-e2e-test resourceVersion: "27560" uid: e2e1c182-96b5-478f-a3a5-b5bef18bc6ed kind: Event lastTimestamp: "2026-07-16T22:24:23Z" message: Started container splitter-graph-76a97 metadata: creationTimestamp: "2026-07-16T22:24:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:24:23Z" name: splitter-graph-76a97-8669877574-rbn4w.18c2e56e0561f68c namespace: kserve-ci-e2e-test resourceVersion: "27596" uid: 74c26b8f-1689-4910-b21e-7c6fd54cb486 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-76a97} kind: Pod name: splitter-graph-76a97-8669877574-rbn4w namespace: kserve-ci-e2e-test resourceVersion: "27560" uid: e2e1c182-96b5-478f-a3a5-b5bef18bc6ed kind: Event lastTimestamp: "2026-07-16T22:32:36Z" message: Stopping container splitter-graph-76a97 metadata: creationTimestamp: "2026-07-16T22:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:32:36Z" name: splitter-graph-76a97-8669877574-rbn4w.18c2e5e0f3d9ac6a namespace: kserve-ci-e2e-test resourceVersion: "31016" uid: 6d3dd37c-d59e-4358-954f-6e0ee27f69cb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:32:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-76a97} kind: Pod name: splitter-graph-76a97-8669877574-rbn4w namespace: kserve-ci-e2e-test resourceVersion: "27560" uid: e2e1c182-96b5-478f-a3a5-b5bef18bc6ed kind: Event lastTimestamp: "2026-07-16T22:33:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T22:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:33:05Z" name: splitter-graph-76a97-8669877574-rbn4w.18c2e5e1b823c0b0 namespace: kserve-ci-e2e-test resourceVersion: "31257" uid: 9eab7cd8-2e9d-4ed5-8fec-d6d8ea49e94d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-76a97-8669877574 namespace: kserve-ci-e2e-test resourceVersion: "27554" uid: deb1a63e-55bf-44d7-b5ff-3f8165c37048 kind: Event lastTimestamp: "2026-07-16T22:24:21Z" message: 'Created pod: splitter-graph-76a97-8669877574-rbn4w' metadata: creationTimestamp: "2026-07-16T22:24:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:21Z" name: splitter-graph-76a97-8669877574.18c2e56dc1b8a55b namespace: kserve-ci-e2e-test resourceVersion: "27561" uid: fe8ceea1-9a06-4c2f-a68d-2bd1251d4e69 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-76a97 namespace: kserve-ci-e2e-test resourceVersion: "27552" uid: f9e29809-a090-4bbf-b503-296ae2530d22 kind: Event lastTimestamp: "2026-07-16T22:24:21Z" message: Scaled up replica set splitter-graph-76a97-8669877574 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:24:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:21Z" name: splitter-graph-76a97.18c2e56dc09cda4d namespace: kserve-ci-e2e-test resourceVersion: "27556" uid: 239c4bb4-4ec4-4f93-b5e0-ce63c86e9b4e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-76a97 namespace: kserve-ci-e2e-test resourceVersion: "27570" uid: d1c08b6c-babb-482c-b00a-77796fd813ad kind: Event lastTimestamp: "2026-07-16T22:24:22Z" message: InferenceGraph [splitter-graph-76a97] is Ready metadata: creationTimestamp: "2026-07-16T22:24:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:24:22Z" name: splitter-graph-76a97.18c2e56dc53f986a namespace: kserve-ci-e2e-test resourceVersion: "27572" uid: f033a613-4170-464d-a406-206b5f546a5b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-76a97 namespace: kserve-ci-e2e-test resourceVersion: "27553" uid: 85307a93-4f68-4a41-8177-f2830a948866 kind: Event lastTimestamp: "2026-07-16T22:24:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:36Z" name: splitter-graph-76a97.18c2e5713f3abe02 namespace: kserve-ci-e2e-test resourceVersion: "27704" uid: b819cc23-3928-446d-befe-6d0de5198d6c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-76a97 namespace: kserve-ci-e2e-test resourceVersion: "27553" uid: 85307a93-4f68-4a41-8177-f2830a948866 kind: Event lastTimestamp: "2026-07-16T22:24:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:36Z" name: splitter-graph-76a97.18c2e5713fcaf66c namespace: kserve-ci-e2e-test resourceVersion: "27707" uid: da5a278b-afe2-44b2-96d0-988989874b8e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-76a97 namespace: kserve-ci-e2e-test resourceVersion: "27703" uid: 85307a93-4f68-4a41-8177-f2830a948866 kind: Event lastTimestamp: "2026-07-16T22:24:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:51Z" name: splitter-graph-76a97.18c2e574bea4dc0a namespace: kserve-ci-e2e-test resourceVersion: "27810" uid: 4e495319-e04d-4e82-8209-86fdcdbbd7c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-76a97 namespace: kserve-ci-e2e-test resourceVersion: "27703" uid: 85307a93-4f68-4a41-8177-f2830a948866 kind: Event lastTimestamp: "2026-07-16T22:24:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:24:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:51Z" name: splitter-graph-76a97.18c2e574bf196a94 namespace: kserve-ci-e2e-test resourceVersion: "27812" uid: d73aed62-13d1-49e3-9e50-39f7b54c1196 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:23:11.516168Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a3d07-5bd98c69cb-ffz2j namespace: kserve-ci-e2e-test resourceVersion: "26751" uid: 14a77efe-01d3-443e-a0f7-82066068fcec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-a3d07-5bd98c69cb-ffz2j to ip-10-0-131-231.ec2.internal metadata: creationTimestamp: "2026-07-16T22:23:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:23:11Z" name: splitter-graph-a3d07-5bd98c69cb-ffz2j.18c2e55d5b2782c7 namespace: kserve-ci-e2e-test resourceVersion: "26756" uid: f47ebe75-954d-468a-92e3-cc2e2b26cb7c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:11Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a3d07-5bd98c69cb-ffz2j namespace: kserve-ci-e2e-test resourceVersion: "26753" uid: 14a77efe-01d3-443e-a0f7-82066068fcec kind: Event lastTimestamp: "2026-07-16T22:23:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-a3d07-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:11Z" name: splitter-graph-a3d07-5bd98c69cb-ffz2j.18c2e55d69a7a47a namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: 261366a1-8e2b-42a8-b987-b20202c2b78e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:12Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a3d07-5bd98c69cb-ffz2j namespace: kserve-ci-e2e-test resourceVersion: "26760" uid: 14a77efe-01d3-443e-a0f7-82066068fcec kind: Event lastTimestamp: "2026-07-16T22:23:12Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:23:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:23:12Z" name: splitter-graph-a3d07-5bd98c69cb-ffz2j.18c2e55d98cfc276 namespace: kserve-ci-e2e-test resourceVersion: "26773" uid: 4f904580-a807-49a4-84d3-e60e51e19047 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a3d07} kind: Pod name: splitter-graph-a3d07-5bd98c69cb-ffz2j namespace: kserve-ci-e2e-test resourceVersion: "26753" uid: 14a77efe-01d3-443e-a0f7-82066068fcec kind: Event lastTimestamp: "2026-07-16T22:23:12Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7e35538adc84c05d9420d87001e9355aeddffd6326bb391010f8e6c1a3c85203" already present on machine metadata: creationTimestamp: "2026-07-16T22:23:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:12Z" name: splitter-graph-a3d07-5bd98c69cb-ffz2j.18c2e55d9b0a1cbf namespace: kserve-ci-e2e-test resourceVersion: "26777" uid: f9c7b660-ee08-4a25-84ca-a799d5eaab99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a3d07} kind: Pod name: splitter-graph-a3d07-5bd98c69cb-ffz2j namespace: kserve-ci-e2e-test resourceVersion: "26753" uid: 14a77efe-01d3-443e-a0f7-82066068fcec kind: Event lastTimestamp: "2026-07-16T22:23:12Z" message: 'Created container: splitter-graph-a3d07' metadata: creationTimestamp: "2026-07-16T22:23:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:12Z" name: splitter-graph-a3d07-5bd98c69cb-ffz2j.18c2e55d9df1db51 namespace: kserve-ci-e2e-test resourceVersion: "26780" uid: a3a3f9f5-80a2-4c50-8421-224ea63a4d02 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a3d07} kind: Pod name: splitter-graph-a3d07-5bd98c69cb-ffz2j namespace: kserve-ci-e2e-test resourceVersion: "26753" uid: 14a77efe-01d3-443e-a0f7-82066068fcec kind: Event lastTimestamp: "2026-07-16T22:23:12Z" message: Started container splitter-graph-a3d07 metadata: creationTimestamp: "2026-07-16T22:23:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:12Z" name: splitter-graph-a3d07-5bd98c69cb-ffz2j.18c2e55d9e6a0fae namespace: kserve-ci-e2e-test resourceVersion: "26781" uid: a3d339a3-6186-4699-afff-c5b795a9119b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a3d07} kind: Pod name: splitter-graph-a3d07-5bd98c69cb-ffz2j namespace: kserve-ci-e2e-test resourceVersion: "26753" uid: 14a77efe-01d3-443e-a0f7-82066068fcec kind: Event lastTimestamp: "2026-07-16T22:23:21Z" message: Stopping container splitter-graph-a3d07 metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:21Z" name: splitter-graph-a3d07-5bd98c69cb-ffz2j.18c2e55fb8ffcea6 namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 3126f518-f8ff-4ef2-bb6b-436aced3212f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:23:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a3d07} kind: Pod name: splitter-graph-a3d07-5bd98c69cb-ffz2j namespace: kserve-ci-e2e-test resourceVersion: "26753" uid: 14a77efe-01d3-443e-a0f7-82066068fcec kind: Event lastTimestamp: "2026-07-16T22:23:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T22:23:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:48Z" name: splitter-graph-a3d07-5bd98c69cb-ffz2j.18c2e560344c231d namespace: kserve-ci-e2e-test resourceVersion: "27271" uid: 17742364-d5c2-417f-bf34-113d5d7700ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-a3d07-5bd98c69cb namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: 1cf512fe-5e4e-4fbe-9af4-7b08e7826fd1 kind: Event lastTimestamp: "2026-07-16T22:23:11Z" message: 'Created pod: splitter-graph-a3d07-5bd98c69cb-ffz2j' metadata: creationTimestamp: "2026-07-16T22:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:11Z" name: splitter-graph-a3d07-5bd98c69cb.18c2e55d5aa2bd5e namespace: kserve-ci-e2e-test resourceVersion: "26754" uid: d22d0b81-20e6-4e5f-a5a8-e69b8bd5dffb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-a3d07 namespace: kserve-ci-e2e-test resourceVersion: "26746" uid: c6345b4a-749f-41bc-a89f-3fc50dd770e7 kind: Event lastTimestamp: "2026-07-16T22:23:11Z" message: Scaled up replica set splitter-graph-a3d07-5bd98c69cb from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:11Z" name: splitter-graph-a3d07.18c2e55d595276c8 namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: 593a534e-c0b5-4c6a-8ab7-7c43d2f88999 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a3d07 namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: bc94f640-ab5f-4678-8bf2-3021026bc1d5 kind: Event lastTimestamp: "2026-07-16T22:23:11Z" message: InferenceGraph [splitter-graph-a3d07] is Ready metadata: creationTimestamp: "2026-07-16T22:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:23:11Z" name: splitter-graph-a3d07.18c2e55d5ffe45a1 namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: 1a07b7fc-b119-4aeb-ae1c-7523b04a75a3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:11:16.231248Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 890b57c2-207d-4f3b-ba55-5607db805fd0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1ab93-predictor-d659cd46b-gcpdg to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:11:16Z" name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg.18c2e4b6d0d9d78f namespace: kserve-ci-e2e-test resourceVersion: "20238" uid: 8fcde97e-23f3-465e-a630-cf5c4e8e23d6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 890b57c2-207d-4f3b-ba55-5607db805fd0 kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1ab93-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:16Z" name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg.18c2e4b6da22002e namespace: kserve-ci-e2e-test resourceVersion: "20267" uid: 23ba6003-6a41-4ccc-afda-ba3863ded233 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg namespace: kserve-ci-e2e-test resourceVersion: "20245" uid: 890b57c2-207d-4f3b-ba55-5607db805fd0 kind: Event lastTimestamp: "2026-07-16T22:11:17Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:11:17Z" name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg.18c2e4b70f860714 namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: 1cf821ef-6846-414a-939d-83a0bd5c4b06 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 890b57c2-207d-4f3b-ba55-5607db805fd0 kind: Event lastTimestamp: "2026-07-16T22:11:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:17Z" name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg.18c2e4b711dd96c8 namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: 292f7b8c-20cc-4482-9756-26b02373faff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 890b57c2-207d-4f3b-ba55-5607db805fd0 kind: Event lastTimestamp: "2026-07-16T22:11:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:17Z" name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg.18c2e4b7152725c5 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 1ddd2062-2bf3-4ebb-85b3-4948117f4214 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 890b57c2-207d-4f3b-ba55-5607db805fd0 kind: Event lastTimestamp: "2026-07-16T22:11:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:17Z" name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg.18c2e4b715a08f28 namespace: kserve-ci-e2e-test resourceVersion: "20295" uid: db7d35a7-7116-4e16-8ac2-f92541ec718d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 890b57c2-207d-4f3b-ba55-5607db805fd0 kind: Event lastTimestamp: "2026-07-16T22:11:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:17Z" name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg.18c2e4b715abc495 namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: 02f32921-138d-4ff5-a260-b2df11ccf8c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 890b57c2-207d-4f3b-ba55-5607db805fd0 kind: Event lastTimestamp: "2026-07-16T22:11:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:17Z" name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg.18c2e4b718f8b937 namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: 8b3170d7-225d-44b0-ab80-87d8ac2468d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 890b57c2-207d-4f3b-ba55-5607db805fd0 kind: Event lastTimestamp: "2026-07-16T22:11:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:11:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:17Z" name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg.18c2e4b7197a1986 namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: fcc7669c-9c7e-412d-aff2-02ee3ec144c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T22:11:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 890b57c2-207d-4f3b-ba55-5607db805fd0 kind: Event lastTimestamp: "2026-07-16T22:12:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:11:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:04Z" name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg.18c2e4b75774b77c namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: 6e181dea-c9ef-4199-baa9-fdecdf1a4bf5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 890b57c2-207d-4f3b-ba55-5607db805fd0 kind: Event lastTimestamp: "2026-07-16T22:12:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:36Z" name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg.18c2e4c97fbab6b3 namespace: kserve-ci-e2e-test resourceVersion: "21179" uid: 721e8e14-f540-44e5-818c-b580f94c009b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 890b57c2-207d-4f3b-ba55-5607db805fd0 kind: Event lastTimestamp: "2026-07-16T22:12:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:36Z" name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg.18c2e4c97fbf7948 namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: 8efeda84-a0cf-42b9-9258-15b34b38d7b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 890b57c2-207d-4f3b-ba55-5607db805fd0 kind: Event lastTimestamp: "2026-07-16T22:12:39Z" message: 'Readiness probe failed: Get "https://10.134.0.45:8643/healthz": dial tcp 10.134.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:12:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:39Z" name: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg.18c2e4ca33a5d0d1 namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: bc4d3cbb-f9f4-4a30-8f00-bda985d47e2c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1ab93-predictor-d659cd46b namespace: kserve-ci-e2e-test resourceVersion: "20202" uid: 01d1f9f7-6798-4a13-a717-8defe8f98115 kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: 'Created pod: success-200-isvc-1ab93-predictor-d659cd46b-gcpdg' metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:11:16Z" name: success-200-isvc-1ab93-predictor-d659cd46b.18c2e4b6cfcda84d namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: 3a4f70f6-decb-4e78-8a4f-c2eaf73bca6c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1ab93-predictor namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 9470f912-c20b-4160-887a-1692cafbc170 kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: Scaled up replica set success-200-isvc-1ab93-predictor-d659cd46b from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:11:16Z" name: success-200-isvc-1ab93-predictor.18c2e4b6ca4478c0 namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: 59576498-e170-4e4b-9554-721ccf6e70ba reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ab93-predictor namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: 254847b1-3e2b-4001-bd82-1f14ce9b797b kind: Event lastTimestamp: "2026-07-16T22:11:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:11:31Z" name: success-200-isvc-1ab93-predictor.18c2e4ba499aef59 namespace: kserve-ci-e2e-test resourceVersion: "20419" uid: 7f082e72-9fe3-48b1-9731-34b7701d69e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ab93-predictor namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: 254847b1-3e2b-4001-bd82-1f14ce9b797b kind: Event lastTimestamp: "2026-07-16T22:11:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:11:31Z" name: success-200-isvc-1ab93-predictor.18c2e4ba4a14053c namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: 2ef6d215-2f8f-4d48-81bf-55433aa509af reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:11:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ab93-predictor namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 254847b1-3e2b-4001-bd82-1f14ce9b797b kind: Event lastTimestamp: "2026-07-16T22:12:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:31Z" name: success-200-isvc-1ab93-predictor.18c2e4bdc8d81435 namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: 6009bc23-5c37-4615-bb7b-5fc22728417e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:11:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ab93-predictor namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 254847b1-3e2b-4001-bd82-1f14ce9b797b kind: Event lastTimestamp: "2026-07-16T22:12:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:31Z" name: success-200-isvc-1ab93-predictor.18c2e4bdc9494eff namespace: kserve-ci-e2e-test resourceVersion: "21092" uid: 34988829-612f-4220-861e-5b6b6f2d84df reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ab93 namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: ec858b02-5b73-4447-847f-a13f4647fb4a kind: Event lastTimestamp: "2026-07-16T22:12:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:12:14Z" name: success-200-isvc-1ab93.18c2e4b6cad8b512 namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: c1503a36-4ca2-4a25-903f-e19d644f3d7e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ab93 namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: ec858b02-5b73-4447-847f-a13f4647fb4a kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-1ab93": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1ab93": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:11:16Z" name: success-200-isvc-1ab93.18c2e4b6cb4e2a5f namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: 2717fcc9-ba78-464a-834e-5f5da5b003b2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ab93 namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: ec858b02-5b73-4447-847f-a13f4647fb4a kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1ab93": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:11:16Z" name: success-200-isvc-1ab93.18c2e4b6cb4e81ec namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: 70436607-f093-4b15-b292-4fd622f5fc28 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ab93 namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: ec858b02-5b73-4447-847f-a13f4647fb4a kind: Event lastTimestamp: "2026-07-16T22:12:14Z" message: InferenceService [success-200-isvc-1ab93] is Ready metadata: creationTimestamp: "2026-07-16T22:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:12:14Z" name: success-200-isvc-1ab93.18c2e4c46aaac6db namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: 1383013a-5af3-499b-b96d-12697bc60b7d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:11:52.125399Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-67446-predictor-58758849c6-zml4x namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 8f981066-32e7-46af-b68c-6a64857028b8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-67446-predictor-58758849c6-zml4x to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:11:52Z" name: success-200-isvc-67446-predictor-58758849c6-zml4x.18c2e4bf2c4f26c0 namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: 8637d3e2-e49a-43d1-b292-942865f5bb5b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-67446-predictor-58758849c6-zml4x namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 8f981066-32e7-46af-b68c-6a64857028b8 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-67446-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:52Z" name: success-200-isvc-67446-predictor-58758849c6-zml4x.18c2e4bf3888ee40 namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: c1fb9eaa-e919-4898-8d46-661aa565dcc9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:53Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-67446-predictor-58758849c6-zml4x namespace: kserve-ci-e2e-test resourceVersion: "20656" uid: 8f981066-32e7-46af-b68c-6a64857028b8 kind: Event lastTimestamp: "2026-07-16T22:11:53Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:11:53Z" name: success-200-isvc-67446-predictor-58758849c6-zml4x.18c2e4bf6ac440cd namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: 5411385f-55be-4189-9d6b-818c05f233b8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67446-predictor-58758849c6-zml4x namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 8f981066-32e7-46af-b68c-6a64857028b8 kind: Event lastTimestamp: "2026-07-16T22:11:53Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:53Z" name: success-200-isvc-67446-predictor-58758849c6-zml4x.18c2e4bf6c262bf9 namespace: kserve-ci-e2e-test resourceVersion: "20710" uid: 5ebc1762-e656-4f7f-b028-bcc6b3fb48de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67446-predictor-58758849c6-zml4x namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 8f981066-32e7-46af-b68c-6a64857028b8 kind: Event lastTimestamp: "2026-07-16T22:11:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:53Z" name: success-200-isvc-67446-predictor-58758849c6-zml4x.18c2e4bf7010b85c namespace: kserve-ci-e2e-test resourceVersion: "20711" uid: 115178a0-2be4-4516-a362-a7f8f047f83b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67446-predictor-58758849c6-zml4x namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 8f981066-32e7-46af-b68c-6a64857028b8 kind: Event lastTimestamp: "2026-07-16T22:11:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:53Z" name: success-200-isvc-67446-predictor-58758849c6-zml4x.18c2e4bf708df7da namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 02d72500-8ad7-49a4-93e2-8e990261eba7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67446-predictor-58758849c6-zml4x namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 8f981066-32e7-46af-b68c-6a64857028b8 kind: Event lastTimestamp: "2026-07-16T22:11:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:53Z" name: success-200-isvc-67446-predictor-58758849c6-zml4x.18c2e4bf709abd88 namespace: kserve-ci-e2e-test resourceVersion: "20713" uid: 77a65eb3-9c31-4590-847a-01b685d847cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67446-predictor-58758849c6-zml4x namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 8f981066-32e7-46af-b68c-6a64857028b8 kind: Event lastTimestamp: "2026-07-16T22:11:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:53Z" name: success-200-isvc-67446-predictor-58758849c6-zml4x.18c2e4bf740eb62b namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 879937d9-02e7-44d1-ba43-c5ce4612df69 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67446-predictor-58758849c6-zml4x namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 8f981066-32e7-46af-b68c-6a64857028b8 kind: Event lastTimestamp: "2026-07-16T22:11:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:53Z" name: success-200-isvc-67446-predictor-58758849c6-zml4x.18c2e4bf748db00e namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: 4f1d1862-bff3-4391-862e-6ab1a63d854d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T22:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67446-predictor-58758849c6-zml4x namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 8f981066-32e7-46af-b68c-6a64857028b8 kind: Event lastTimestamp: "2026-07-16T22:12:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:40Z" name: success-200-isvc-67446-predictor-58758849c6-zml4x.18c2e4bfc5a4b50b namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: 8a22cbdc-edd5-4278-8919-cf63ce922c29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-67446-predictor-58758849c6-zml4x namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 8f981066-32e7-46af-b68c-6a64857028b8 kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:12Z" name: success-200-isvc-67446-predictor-58758849c6-zml4x.18c2e4d1de43645f namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 495de8e5-9fce-480c-9de4-e32022d5a09f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67446-predictor-58758849c6-zml4x namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 8f981066-32e7-46af-b68c-6a64857028b8 kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:12Z" name: success-200-isvc-67446-predictor-58758849c6-zml4x.18c2e4d1de43c44c namespace: kserve-ci-e2e-test resourceVersion: "21686" uid: f30b4055-7a84-49ca-bbea-ccd508ce880e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-67446-predictor-58758849c6-zml4x namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 8f981066-32e7-46af-b68c-6a64857028b8 kind: Event lastTimestamp: "2026-07-16T22:13:15Z" message: 'Readiness probe failed: Get "https://10.134.0.47:8643/healthz": dial tcp 10.134.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:15Z" name: success-200-isvc-67446-predictor-58758849c6-zml4x.18c2e4d2a1e69e27 namespace: kserve-ci-e2e-test resourceVersion: "21796" uid: 4bad195f-41ad-4508-b0a3-7ecb9a4906fd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-67446-predictor-58758849c6 namespace: kserve-ci-e2e-test resourceVersion: "20634" uid: 0f236876-3137-4c41-b1a2-6fff7d99757b kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: 'Created pod: success-200-isvc-67446-predictor-58758849c6-zml4x' metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:11:52Z" name: success-200-isvc-67446-predictor-58758849c6.18c2e4bf2be50274 namespace: kserve-ci-e2e-test resourceVersion: "20649" uid: b6d8f2fb-82f7-4aab-859b-8433e6de9616 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-67446-predictor namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: 931b598d-2260-4398-95d5-392301d57a68 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: Scaled up replica set success-200-isvc-67446-predictor-58758849c6 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:11:52Z" name: success-200-isvc-67446-predictor.18c2e4bf29b3868a namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: fcc58ab7-4115-4890-84cc-d3ae7acb2057 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-67446-predictor namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 2d075ecd-3086-417b-b6d9-c9dbcae5c52a kind: Event lastTimestamp: "2026-07-16T22:12:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:07Z" name: success-200-isvc-67446-predictor.18c2e4c2a850c6d5 namespace: kserve-ci-e2e-test resourceVersion: "20849" uid: 92b27ce8-934a-4c42-abad-2de7e4fc16cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-67446-predictor namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 2d075ecd-3086-417b-b6d9-c9dbcae5c52a kind: Event lastTimestamp: "2026-07-16T22:12:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:07Z" name: success-200-isvc-67446-predictor.18c2e4c2a8c2255e namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: d8ccebe5-1c8c-4862-b745-8ac2dbc10b83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:12:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-67446-predictor namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 2d075ecd-3086-417b-b6d9-c9dbcae5c52a kind: Event lastTimestamp: "2026-07-16T22:13:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:07Z" name: success-200-isvc-67446-predictor.18c2e4c6279387db namespace: kserve-ci-e2e-test resourceVersion: "21583" uid: 43268086-4a7f-4406-9f4e-fc55ffde1b4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:12:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-67446-predictor namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 2d075ecd-3086-417b-b6d9-c9dbcae5c52a kind: Event lastTimestamp: "2026-07-16T22:13:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:07Z" name: success-200-isvc-67446-predictor.18c2e4c628016d9c namespace: kserve-ci-e2e-test resourceVersion: "21584" uid: c9493839-7d57-4497-bd21-8a4109077566 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-67446 namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: e57f49ae-5151-467c-a40f-f644c68a44d2 kind: Event lastTimestamp: "2026-07-16T22:12:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:12:50Z" name: success-200-isvc-67446.18c2e4bf29cabf7f namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: 18650e2c-992e-420a-ae42-d64836849d29 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-67446 namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: e57f49ae-5151-467c-a40f-f644c68a44d2 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: 'Failed to update status for InferenceService "success-200-isvc-67446": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-67446": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:11:52Z" name: success-200-isvc-67446.18c2e4bf2a4b4371 namespace: kserve-ci-e2e-test resourceVersion: "20640" uid: a2ae89ff-8de6-4a26-bacc-a127be24149e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-67446 namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: e57f49ae-5151-467c-a40f-f644c68a44d2 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-67446": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:11:52Z" name: success-200-isvc-67446.18c2e4bf2a4ba16c namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 64af8137-1ae3-4d39-9033-2c677647be84 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-67446 namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: e57f49ae-5151-467c-a40f-f644c68a44d2 kind: Event lastTimestamp: "2026-07-16T22:12:50Z" message: InferenceService [success-200-isvc-67446] is Ready metadata: creationTimestamp: "2026-07-16T22:12:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:12:50Z" name: success-200-isvc-67446.18c2e4ccd6b6ba97 namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 67dbd8f2-ea95-4a97-8e55-6318542adcc1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:01:51.395924Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw namespace: kserve-ci-e2e-test resourceVersion: "15754" uid: d6b29823-fd41-4147-959f-e09b96ad0567 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-69517-predictor-85bc9bd758-gtdkw to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw.18c2e4334e0ad0e8 namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: 08e83a08-b9eb-495f-ad8d-2b625212afe1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: d6b29823-fd41-4147-959f-e09b96ad0567 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw.18c2e43368ed3e7d namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 4f545987-6363-49be-8348-703713383ec2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: d6b29823-fd41-4147-959f-e09b96ad0567 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw.18c2e4336a48dd6d namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 85a76a3d-b1b9-4028-967b-cd88106850ca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: d6b29823-fd41-4147-959f-e09b96ad0567 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw.18c2e4336d976098 namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: 3d1f14d1-5b8a-4ce8-87a5-759f6692628f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: d6b29823-fd41-4147-959f-e09b96ad0567 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw.18c2e4336e4038c6 namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 6857fd91-35e3-4ec8-b15f-536c5bff1418 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: d6b29823-fd41-4147-959f-e09b96ad0567 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw.18c2e4336e4b67a9 namespace: kserve-ci-e2e-test resourceVersion: "15812" uid: 3d9163e1-774f-4afa-9292-5817fab12e85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: d6b29823-fd41-4147-959f-e09b96ad0567 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw.18c2e4337146364c namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 5e1184f0-a87f-4069-b586-3621d4940d79 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: d6b29823-fd41-4147-959f-e09b96ad0567 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw.18c2e43371c5cd42 namespace: kserve-ci-e2e-test resourceVersion: "15815" uid: 8aae89d8-1b9a-4a9f-9ac4-2c1a53c9b6b4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-16T22:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: d6b29823-fd41-4147-959f-e09b96ad0567 kind: Event lastTimestamp: "2026-07-16T22:11:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:19Z" name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw.18c2e43392c10385 namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: 956acb50-1e1b-4cd6-9ff0-a4abc5cf588b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: d6b29823-fd41-4147-959f-e09b96ad0567 kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:16Z" name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw.18c2e4b6c948878b namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: 8085f1df-6e3e-4415-a79b-57696ab00138 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: d6b29823-fd41-4147-959f-e09b96ad0567 kind: Event lastTimestamp: "2026-07-16T22:11:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:16Z" name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw.18c2e4b6c949cacb namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: bfbe1c7b-98f6-4834-9b31-4e42276a8648 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: d6b29823-fd41-4147-959f-e09b96ad0567 kind: Event lastTimestamp: "2026-07-16T22:11:19Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:11:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:19Z" name: success-200-isvc-69517-predictor-85bc9bd758-gtdkw.18c2e4b79793ad91 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: ba7c3b83-e93c-4494-a4e2-d1e337b5dc3c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-69517-predictor-85bc9bd758 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: d211a138-1071-47b9-b01b-a1bec0c5a36f kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: 'Created pod: success-200-isvc-69517-predictor-85bc9bd758-gtdkw' metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-69517-predictor-85bc9bd758.18c2e4334d7dcda8 namespace: kserve-ci-e2e-test resourceVersion: "15757" uid: 087c19a3-46ce-4096-9e2c-7ff793e4705f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-69517-predictor namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 3bb40622-16d7-4e73-bb80-a2a179b68cfc kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: Scaled up replica set success-200-isvc-69517-predictor-85bc9bd758 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-69517-predictor.18c2e43348cadf12 namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: 431503f6-db73-4327-9785-8507d8147ec3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:02:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-69517-predictor namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: 5d606ebe-9480-4608-828b-4925cf89f39c kind: Event lastTimestamp: "2026-07-16T22:02:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:21Z" name: success-200-isvc-69517-predictor.18c2e436c79a885d namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 3d04de57-4975-4f23-ac04-085ee72e9548 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:02:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-69517-predictor namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: 5d606ebe-9480-4608-828b-4925cf89f39c kind: Event lastTimestamp: "2026-07-16T22:02:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:21Z" name: success-200-isvc-69517-predictor.18c2e436c80e92f9 namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: 442b9e5b-7824-4c78-be95-fcaf090e61d2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:02:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-69517-predictor namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 5d606ebe-9480-4608-828b-4925cf89f39c kind: Event lastTimestamp: "2026-07-16T22:03:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:06Z" name: success-200-isvc-69517-predictor.18c2e43dc5bebd24 namespace: kserve-ci-e2e-test resourceVersion: "16721" uid: e9e57b03-db80-4d0f-ba0a-580f35894f28 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:02:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-69517-predictor namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 5d606ebe-9480-4608-828b-4925cf89f39c kind: Event lastTimestamp: "2026-07-16T22:03:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:06Z" name: success-200-isvc-69517-predictor.18c2e43dc63a0dc6 namespace: kserve-ci-e2e-test resourceVersion: "16722" uid: ecd686a9-ca6f-449d-a655-c0a088383162 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-69517 namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: ee392a3f-290c-4ba4-a0c9-f234276002d9 kind: Event lastTimestamp: "2026-07-16T22:02:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:02:49Z" name: success-200-isvc-69517.18c2e4334a03c7f3 namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 80456084-03ff-4773-8728-3f53f52813ee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-69517 namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: ee392a3f-290c-4ba4-a0c9-f234276002d9 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-69517": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-69517": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-69517.18c2e4334ab2ea39 namespace: kserve-ci-e2e-test resourceVersion: "15747" uid: 821a6fc2-e9fb-424d-92aa-7772a6fa597c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-69517 namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: ee392a3f-290c-4ba4-a0c9-f234276002d9 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-69517": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-69517.18c2e4334ab331ef namespace: kserve-ci-e2e-test resourceVersion: "15750" uid: 3e063864-6774-435e-a5bb-ef84f8eb7c05 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-69517 namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: ee392a3f-290c-4ba4-a0c9-f234276002d9 kind: Event lastTimestamp: "2026-07-16T22:02:49Z" message: InferenceService [success-200-isvc-69517] is Ready metadata: creationTimestamp: "2026-07-16T22:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:02:49Z" name: success-200-isvc-69517.18c2e440deee1fb3 namespace: kserve-ci-e2e-test resourceVersion: "16549" uid: 47998f71-0c5e-4b87-b728-1b391b99b77e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:23:21.905270Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99 namespace: kserve-ci-e2e-test resourceVersion: "26912" uid: 94458687-2e60-41ce-ab4e-30e3412d94d2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-76a97-predictor-7d9f876bfd-6nl99 to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:23:21Z" name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99.18c2e55fc664a208 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 71eb6ef5-eaf6-4ccb-81f8-21e0f752f1b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99 namespace: kserve-ci-e2e-test resourceVersion: "26922" uid: 94458687-2e60-41ce-ab4e-30e3412d94d2 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: Add eth0 [10.134.0.57/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:23:22Z" name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99.18c2e55fe154421c namespace: kserve-ci-e2e-test resourceVersion: "26975" uid: 60a13e16-da89-4fa8-95b8-af4fa7bdeb82 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99 namespace: kserve-ci-e2e-test resourceVersion: "26913" uid: 94458687-2e60-41ce-ab4e-30e3412d94d2 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:22Z" name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99.18c2e55fe2e69a3e namespace: kserve-ci-e2e-test resourceVersion: "26977" uid: 9f6f0875-3d4f-41de-b05b-f63bc3d854e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99 namespace: kserve-ci-e2e-test resourceVersion: "26913" uid: 94458687-2e60-41ce-ab4e-30e3412d94d2 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:22Z" name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99.18c2e55fe69ba14e namespace: kserve-ci-e2e-test resourceVersion: "26981" uid: 8e7795b1-b096-4277-b8a8-1ea39dbcd2f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99 namespace: kserve-ci-e2e-test resourceVersion: "26913" uid: 94458687-2e60-41ce-ab4e-30e3412d94d2 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:22Z" name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99.18c2e55fe7237aa7 namespace: kserve-ci-e2e-test resourceVersion: "26982" uid: c73d8b70-75ef-426a-bf1a-88fd2425c6fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99 namespace: kserve-ci-e2e-test resourceVersion: "26913" uid: 94458687-2e60-41ce-ab4e-30e3412d94d2 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:22Z" name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99.18c2e55fe731dc49 namespace: kserve-ci-e2e-test resourceVersion: "26984" uid: 20efc23e-7694-4629-8b53-6f03ab4181c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99 namespace: kserve-ci-e2e-test resourceVersion: "26913" uid: 94458687-2e60-41ce-ab4e-30e3412d94d2 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:22Z" name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99.18c2e55feb2541c2 namespace: kserve-ci-e2e-test resourceVersion: "26988" uid: b639010c-9c18-48e5-9938-592f4c439624 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99 namespace: kserve-ci-e2e-test resourceVersion: "26913" uid: 94458687-2e60-41ce-ab4e-30e3412d94d2 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:22Z" name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99.18c2e55febaef0f3 namespace: kserve-ci-e2e-test resourceVersion: "26989" uid: 61252eb4-703e-49ff-a5e8-ca44da527d69 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T22:23:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99 namespace: kserve-ci-e2e-test resourceVersion: "26913" uid: 94458687-2e60-41ce-ab4e-30e3412d94d2 kind: Event lastTimestamp: "2026-07-16T22:32:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.57:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:23:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:32:40Z" name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99.18c2e5605e85c5eb namespace: kserve-ci-e2e-test resourceVersion: "31078" uid: 03a3c879-5005-4900-8328-bf8dd4a8b4df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99 namespace: kserve-ci-e2e-test resourceVersion: "26913" uid: 94458687-2e60-41ce-ab4e-30e3412d94d2 kind: Event lastTimestamp: "2026-07-16T22:32:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:32:36Z" name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99.18c2e5e0fc423a5b namespace: kserve-ci-e2e-test resourceVersion: "31035" uid: 91dc9f3d-1db1-4208-bc2a-a6e3a3863c87 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:32:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99 namespace: kserve-ci-e2e-test resourceVersion: "26913" uid: 94458687-2e60-41ce-ab4e-30e3412d94d2 kind: Event lastTimestamp: "2026-07-16T22:32:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:32:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:32:36Z" name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99.18c2e5e0fc42a524 namespace: kserve-ci-e2e-test resourceVersion: "31037" uid: 3f17a872-2b33-4eea-8323-d19e5b3eb352 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:32:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99 namespace: kserve-ci-e2e-test resourceVersion: "26913" uid: 94458687-2e60-41ce-ab4e-30e3412d94d2 kind: Event lastTimestamp: "2026-07-16T22:32:40Z" message: 'Readiness probe failed: Get "https://10.134.0.57:8643/healthz": dial tcp 10.134.0.57:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:32:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:32:40Z" name: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99.18c2e5e1d2f5e29a namespace: kserve-ci-e2e-test resourceVersion: "31077" uid: 4ee3a8f3-6e7c-49a5-a8e0-026515d0403d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-76a97-predictor-7d9f876bfd namespace: kserve-ci-e2e-test resourceVersion: "26903" uid: 42f0a0a5-fce5-40a9-80f3-aed8b6ef188b kind: Event lastTimestamp: "2026-07-16T22:23:21Z" message: 'Created pod: success-200-isvc-76a97-predictor-7d9f876bfd-6nl99' metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:21Z" name: success-200-isvc-76a97-predictor-7d9f876bfd.18c2e55fc5e3a9dc namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: 4f947e77-80e3-4448-b5dd-9e482c6ae7b5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-76a97-predictor namespace: kserve-ci-e2e-test resourceVersion: "26900" uid: 6f2619b2-d923-49c9-9153-5ae2649d1320 kind: Event lastTimestamp: "2026-07-16T22:23:21Z" message: Scaled up replica set success-200-isvc-76a97-predictor-7d9f876bfd from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:21Z" name: success-200-isvc-76a97-predictor.18c2e55fc3c8dfda namespace: kserve-ci-e2e-test resourceVersion: "26906" uid: eecfb9f2-fb5c-40d2-89a5-36e03e14ea80 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:23:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-76a97-predictor namespace: kserve-ci-e2e-test resourceVersion: "26902" uid: 01cf86ed-8dd9-4542-8dae-7eb6bd48dcec kind: Event lastTimestamp: "2026-07-16T22:23:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:51Z" name: success-200-isvc-76a97-predictor.18c2e56342ec46db namespace: kserve-ci-e2e-test resourceVersion: "27301" uid: e926591f-23f4-4b1e-8d37-d8eb231661db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:23:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-76a97-predictor namespace: kserve-ci-e2e-test resourceVersion: "26902" uid: 01cf86ed-8dd9-4542-8dae-7eb6bd48dcec kind: Event lastTimestamp: "2026-07-16T22:23:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:23:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:51Z" name: success-200-isvc-76a97-predictor.18c2e5634365ef6f namespace: kserve-ci-e2e-test resourceVersion: "27302" uid: 0e7ffed1-2d97-430b-9f8b-275e2f9fc470 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:24:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-76a97-predictor namespace: kserve-ci-e2e-test resourceVersion: "27140" uid: 01cf86ed-8dd9-4542-8dae-7eb6bd48dcec kind: Event lastTimestamp: "2026-07-16T22:24:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:36Z" name: success-200-isvc-76a97-predictor.18c2e56a41547a21 namespace: kserve-ci-e2e-test resourceVersion: "27705" uid: 38ec5920-8541-4713-92dd-b9330b306570 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:24:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-76a97-predictor namespace: kserve-ci-e2e-test resourceVersion: "27140" uid: 01cf86ed-8dd9-4542-8dae-7eb6bd48dcec kind: Event lastTimestamp: "2026-07-16T22:24:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:24:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:36Z" name: success-200-isvc-76a97-predictor.18c2e56a41c38c4b namespace: kserve-ci-e2e-test resourceVersion: "27706" uid: c4332362-ef4c-42b0-95ad-23598ee8fd64 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-76a97 namespace: kserve-ci-e2e-test resourceVersion: "26879" uid: c9b84cbd-ffa5-40f3-bf53-bbc423e3f589 kind: Event lastTimestamp: "2026-07-16T22:24:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:24:10Z" name: success-200-isvc-76a97.18c2e55fc3f2b757 namespace: kserve-ci-e2e-test resourceVersion: "27462" uid: 33c46b2c-56c3-4626-9a92-33dcf970ccd8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-76a97 namespace: kserve-ci-e2e-test resourceVersion: "26879" uid: c9b84cbd-ffa5-40f3-bf53-bbc423e3f589 kind: Event lastTimestamp: "2026-07-16T22:23:21Z" message: 'Failed to update status for InferenceService "success-200-isvc-76a97": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-76a97": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:23:21Z" name: success-200-isvc-76a97.18c2e55fc4aa11b9 namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: b24831e3-7ea0-4906-80c0-8ceee9896915 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-76a97 namespace: kserve-ci-e2e-test resourceVersion: "26879" uid: c9b84cbd-ffa5-40f3-bf53-bbc423e3f589 kind: Event lastTimestamp: "2026-07-16T22:23:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-76a97": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:23:21Z" name: success-200-isvc-76a97.18c2e55fc4aa6475 namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: ab265c0b-9e2a-46a0-83f5-8baf8af6f581 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:24:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-76a97 namespace: kserve-ci-e2e-test resourceVersion: "27459" uid: c9b84cbd-ffa5-40f3-bf53-bbc423e3f589 kind: Event lastTimestamp: "2026-07-16T22:24:10Z" message: InferenceService [success-200-isvc-76a97] is Ready metadata: creationTimestamp: "2026-07-16T22:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:24:10Z" name: success-200-isvc-76a97.18c2e56b1b376010 namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: 427f8f45-e893-43ec-94b6-6b9672f90b24 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:02:37.259916Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-77f35-predictor-766f77d998-qfjbd namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 8832ac4b-a3b2-4a46-82ca-c9049562d2e7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-77f35-predictor-766f77d998-qfjbd to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:02:37Z" name: success-200-isvc-77f35-predictor-766f77d998-qfjbd.18c2e43dfbbfb468 namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: 0f7cc094-7301-417b-8972-e460d1501ae8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-77f35-predictor-766f77d998-qfjbd namespace: kserve-ci-e2e-test resourceVersion: "16345" uid: 8832ac4b-a3b2-4a46-82ca-c9049562d2e7 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:02:37Z" name: success-200-isvc-77f35-predictor-766f77d998-qfjbd.18c2e43e168b3ed8 namespace: kserve-ci-e2e-test resourceVersion: "16402" uid: c70da6e7-28fe-46d3-ab0e-7e68dadc18a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77f35-predictor-766f77d998-qfjbd namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 8832ac4b-a3b2-4a46-82ca-c9049562d2e7 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:37Z" name: success-200-isvc-77f35-predictor-766f77d998-qfjbd.18c2e43e183d154c namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: 47ee811f-c423-4627-813c-cf160a49bbe8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77f35-predictor-766f77d998-qfjbd namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 8832ac4b-a3b2-4a46-82ca-c9049562d2e7 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:37Z" name: success-200-isvc-77f35-predictor-766f77d998-qfjbd.18c2e43e1b6862a4 namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: 12555151-e218-46eb-9413-de2038256075 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77f35-predictor-766f77d998-qfjbd namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 8832ac4b-a3b2-4a46-82ca-c9049562d2e7 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:37Z" name: success-200-isvc-77f35-predictor-766f77d998-qfjbd.18c2e43e1be18e8c namespace: kserve-ci-e2e-test resourceVersion: "16410" uid: 2269b70b-17e2-484c-b06f-9d9294cf892c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77f35-predictor-766f77d998-qfjbd namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 8832ac4b-a3b2-4a46-82ca-c9049562d2e7 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:37Z" name: success-200-isvc-77f35-predictor-766f77d998-qfjbd.18c2e43e1beca1cf namespace: kserve-ci-e2e-test resourceVersion: "16411" uid: 286499ee-a3c5-4889-92fd-d6c1386a2b40 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77f35-predictor-766f77d998-qfjbd namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 8832ac4b-a3b2-4a46-82ca-c9049562d2e7 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:37Z" name: success-200-isvc-77f35-predictor-766f77d998-qfjbd.18c2e43e1f2ebba2 namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: 9356f607-d896-4ed4-b7c9-9406a99e4754 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77f35-predictor-766f77d998-qfjbd namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 8832ac4b-a3b2-4a46-82ca-c9049562d2e7 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:37Z" name: success-200-isvc-77f35-predictor-766f77d998-qfjbd.18c2e43e1fc7ffb9 namespace: kserve-ci-e2e-test resourceVersion: "16413" uid: 6527893e-2d3c-419c-bc04-8ea8b23ef1e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-16T22:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77f35-predictor-766f77d998-qfjbd namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 8832ac4b-a3b2-4a46-82ca-c9049562d2e7 kind: Event lastTimestamp: "2026-07-16T22:11:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:55Z" name: success-200-isvc-77f35-predictor-766f77d998-qfjbd.18c2e43e589365be namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 0a08eff2-68a9-4d48-84c1-2033854ad634 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-77f35-predictor-766f77d998-qfjbd namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 8832ac4b-a3b2-4a46-82ca-c9049562d2e7 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:52Z" name: success-200-isvc-77f35-predictor-766f77d998-qfjbd.18c2e4bf26e9ac59 namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 42c67802-c6c4-4f25-bb24-d6e7fdfce810 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77f35-predictor-766f77d998-qfjbd namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 8832ac4b-a3b2-4a46-82ca-c9049562d2e7 kind: Event lastTimestamp: "2026-07-16T22:11:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:52Z" name: success-200-isvc-77f35-predictor-766f77d998-qfjbd.18c2e4bf26ea26a1 namespace: kserve-ci-e2e-test resourceVersion: "20625" uid: 9fc246a2-5b5f-4430-b41d-94b6efc130e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-77f35-predictor-766f77d998-qfjbd namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 8832ac4b-a3b2-4a46-82ca-c9049562d2e7 kind: Event lastTimestamp: "2026-07-16T22:11:55Z" message: 'Readiness probe failed: Get "https://10.134.0.43:8643/healthz": dial tcp 10.134.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:11:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:55Z" name: success-200-isvc-77f35-predictor-766f77d998-qfjbd.18c2e4c0091b9b50 namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: f3c643c9-3018-43ad-8564-18b1fb4a65b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-77f35-predictor-766f77d998 namespace: kserve-ci-e2e-test resourceVersion: "16307" uid: c8dfb914-8ae0-405a-8d07-5462d168ba1c kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: 'Created pod: success-200-isvc-77f35-predictor-766f77d998-qfjbd' metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:37Z" name: success-200-isvc-77f35-predictor-766f77d998.18c2e43dfaf61596 namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 6a1bd46e-8dd3-41d1-9a85-0e9e4b670a5f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-77f35-predictor namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 42169304-f90d-4a4b-ab5b-7906a0388908 kind: Event lastTimestamp: "2026-07-16T22:02:37Z" message: Scaled up replica set success-200-isvc-77f35-predictor-766f77d998 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:37Z" name: success-200-isvc-77f35-predictor.18c2e43df70a8803 namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: 97926495-b7be-43e8-9add-250047eb7143 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-77f35-predictor namespace: kserve-ci-e2e-test resourceVersion: "16308" uid: 28768e2c-caa8-476d-97d5-544e5e7d46af kind: Event lastTimestamp: "2026-07-16T22:02:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:52Z" name: success-200-isvc-77f35-predictor.18c2e44175bdaa9c namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: bb1b29a2-be5c-4b18-b88a-97698fa3a70f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:02:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-77f35-predictor namespace: kserve-ci-e2e-test resourceVersion: "16308" uid: 28768e2c-caa8-476d-97d5-544e5e7d46af kind: Event lastTimestamp: "2026-07-16T22:02:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:02:52Z" name: success-200-isvc-77f35-predictor.18c2e4417603c605 namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: 2614acf3-a692-4baa-b42e-d169f2925a53 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:03:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-77f35-predictor namespace: kserve-ci-e2e-test resourceVersion: "16575" uid: 28768e2c-caa8-476d-97d5-544e5e7d46af kind: Event lastTimestamp: "2026-07-16T22:03:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:52Z" name: success-200-isvc-77f35-predictor.18c2e444f4cf106e namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: 71673b0e-3193-40f3-b8d5-7546936fcbd0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:03:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-77f35-predictor namespace: kserve-ci-e2e-test resourceVersion: "16575" uid: 28768e2c-caa8-476d-97d5-544e5e7d46af kind: Event lastTimestamp: "2026-07-16T22:03:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:52Z" name: success-200-isvc-77f35-predictor.18c2e444f53eba3f namespace: kserve-ci-e2e-test resourceVersion: "17136" uid: 1216aaac-90f5-49ef-b4f9-353101ec7751 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:02:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-77f35 namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: 575e19f0-4d32-4085-9934-8fe33ce5c3a9 kind: Event lastTimestamp: "2026-07-16T22:03:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:03:26Z" name: success-200-isvc-77f35.18c2e43df7a7949d namespace: kserve-ci-e2e-test resourceVersion: "16892" uid: 7187b5df-0768-4374-93b4-f6968df12d02 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-77f35 namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 575e19f0-4d32-4085-9934-8fe33ce5c3a9 kind: Event lastTimestamp: "2026-07-16T22:03:25Z" message: InferenceService [success-200-isvc-77f35] is Ready metadata: creationTimestamp: "2026-07-16T22:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:03:25Z" name: success-200-isvc-77f35.18c2e44950c8cba2 namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: 1fab9732-cffa-4a0d-a18e-98dbd52d20ea reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:22:37.452226Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-93462-predictor-759cb758b6-4cnb7 namespace: kserve-ci-e2e-test resourceVersion: "26344" uid: 3e2c3c09-dd80-4362-9c24-6b7bb097e018 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-93462-predictor-759cb758b6-4cnb7 to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:22:37Z" name: success-200-isvc-93462-predictor-759cb758b6-4cnb7.18c2e5556cc90427 namespace: kserve-ci-e2e-test resourceVersion: "26350" uid: d43a7111-4787-4cac-8825-e7e067198e59 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-93462-predictor-759cb758b6-4cnb7 namespace: kserve-ci-e2e-test resourceVersion: "26348" uid: 3e2c3c09-dd80-4362-9c24-6b7bb097e018 kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-93462-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:37Z" name: success-200-isvc-93462-predictor-759cb758b6-4cnb7.18c2e5557cdb9998 namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: 391a6119-bba1-42d8-8ea8-17c08df82375 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-93462-predictor-759cb758b6-4cnb7 namespace: kserve-ci-e2e-test resourceVersion: "26355" uid: 3e2c3c09-dd80-4362-9c24-6b7bb097e018 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: Add eth0 [10.134.0.55/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:22:38Z" name: success-200-isvc-93462-predictor-759cb758b6-4cnb7.18c2e555ab334216 namespace: kserve-ci-e2e-test resourceVersion: "26421" uid: 93522a99-ce37-47a4-860d-9f472fb9c3ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-93462-predictor-759cb758b6-4cnb7 namespace: kserve-ci-e2e-test resourceVersion: "26348" uid: 3e2c3c09-dd80-4362-9c24-6b7bb097e018 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:38Z" name: success-200-isvc-93462-predictor-759cb758b6-4cnb7.18c2e555acc0ba9e namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 3f541ae2-f28b-4bbb-a44b-9f9b869312ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-93462-predictor-759cb758b6-4cnb7 namespace: kserve-ci-e2e-test resourceVersion: "26348" uid: 3e2c3c09-dd80-4362-9c24-6b7bb097e018 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:38Z" name: success-200-isvc-93462-predictor-759cb758b6-4cnb7.18c2e555b0a20c23 namespace: kserve-ci-e2e-test resourceVersion: "26426" uid: 0b7921c6-ccc0-464b-b92c-f7d29599f64e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-93462-predictor-759cb758b6-4cnb7 namespace: kserve-ci-e2e-test resourceVersion: "26348" uid: 3e2c3c09-dd80-4362-9c24-6b7bb097e018 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:38Z" name: success-200-isvc-93462-predictor-759cb758b6-4cnb7.18c2e555b11fd2a1 namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 945bd741-299c-4545-b1c8-63b6d8d6239f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-93462-predictor-759cb758b6-4cnb7 namespace: kserve-ci-e2e-test resourceVersion: "26348" uid: 3e2c3c09-dd80-4362-9c24-6b7bb097e018 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:38Z" name: success-200-isvc-93462-predictor-759cb758b6-4cnb7.18c2e555b12c0f5c namespace: kserve-ci-e2e-test resourceVersion: "26428" uid: 8bb11082-fa6c-4f73-8efe-99b83bf9d6ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-93462-predictor-759cb758b6-4cnb7 namespace: kserve-ci-e2e-test resourceVersion: "26348" uid: 3e2c3c09-dd80-4362-9c24-6b7bb097e018 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:38Z" name: success-200-isvc-93462-predictor-759cb758b6-4cnb7.18c2e555b4eed90b namespace: kserve-ci-e2e-test resourceVersion: "26430" uid: f6bdf023-20c6-45e2-891e-13c4f931b286 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-93462-predictor-759cb758b6-4cnb7 namespace: kserve-ci-e2e-test resourceVersion: "26348" uid: 3e2c3c09-dd80-4362-9c24-6b7bb097e018 kind: Event lastTimestamp: "2026-07-16T22:22:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:22:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:38Z" name: success-200-isvc-93462-predictor-759cb758b6-4cnb7.18c2e555b58aabd5 namespace: kserve-ci-e2e-test resourceVersion: "26431" uid: 57a70dc4-c76c-488f-98d7-e7ab55fd61eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T22:22:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-93462-predictor-759cb758b6-4cnb7 namespace: kserve-ci-e2e-test resourceVersion: "26348" uid: 3e2c3c09-dd80-4362-9c24-6b7bb097e018 kind: Event lastTimestamp: "2026-07-16T22:23:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.55:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:25Z" name: success-200-isvc-93462-predictor-759cb758b6-4cnb7.18c2e555d437b504 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 0a0f07be-73da-43ab-a761-2fb66438abbc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:40:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-93462-predictor-759cb758b6-4cnb7 namespace: kserve-ci-e2e-test resourceVersion: "26348" uid: 3e2c3c09-dd80-4362-9c24-6b7bb097e018 kind: Event lastTimestamp: "2026-07-16T22:40:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:40:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:40:07Z" name: success-200-isvc-93462-predictor-759cb758b6-4cnb7.18c2e649d3017daf namespace: kserve-ci-e2e-test resourceVersion: "34175" uid: 374209bd-8779-4f53-b96c-5d7561fae154 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:40:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-93462-predictor-759cb758b6-4cnb7 namespace: kserve-ci-e2e-test resourceVersion: "26348" uid: 3e2c3c09-dd80-4362-9c24-6b7bb097e018 kind: Event lastTimestamp: "2026-07-16T22:40:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:40:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:40:07Z" name: success-200-isvc-93462-predictor-759cb758b6-4cnb7.18c2e649d3020a4f namespace: kserve-ci-e2e-test resourceVersion: "34176" uid: b83fa2d1-37fe-44ce-aa67-0704b9cb9049 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:40:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-93462-predictor-759cb758b6-4cnb7 namespace: kserve-ci-e2e-test resourceVersion: "26348" uid: 3e2c3c09-dd80-4362-9c24-6b7bb097e018 kind: Event lastTimestamp: "2026-07-16T22:40:10Z" message: 'Readiness probe failed: Get "https://10.134.0.55:8643/healthz": dial tcp 10.134.0.55:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:40:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:40:10Z" name: success-200-isvc-93462-predictor-759cb758b6-4cnb7.18c2e64a88ef7d8a namespace: kserve-ci-e2e-test resourceVersion: "34230" uid: 4a148d52-a655-47a1-ae30-9efdef1831e7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-93462-predictor-759cb758b6 namespace: kserve-ci-e2e-test resourceVersion: "26330" uid: a606c045-5b85-4c1c-8c22-4f89081bc642 kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: 'Created pod: success-200-isvc-93462-predictor-759cb758b6-4cnb7' metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:22:37Z" name: success-200-isvc-93462-predictor-759cb758b6.18c2e5556b8802f8 namespace: kserve-ci-e2e-test resourceVersion: "26345" uid: 293a0428-a1cf-4774-bb0e-12625b8ad24f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-93462-predictor namespace: kserve-ci-e2e-test resourceVersion: "26325" uid: 53911826-7357-44b6-a510-21e54b4e0b69 kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: Scaled up replica set success-200-isvc-93462-predictor-759cb758b6 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:22:37Z" name: success-200-isvc-93462-predictor.18c2e55567715eea namespace: kserve-ci-e2e-test resourceVersion: "26334" uid: 4abac02a-a2a8-440c-841e-6b434b52f995 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:22:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-93462-predictor namespace: kserve-ci-e2e-test resourceVersion: "26331" uid: 9e8cf8e2-1c1d-4f69-81c6-30df6446be01 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:22:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:22Z" name: success-200-isvc-93462-predictor.18c2e558e63cd6f9 namespace: kserve-ci-e2e-test resourceVersion: "26978" uid: f449ba27-91da-4459-b064-aaafad14a6ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:22:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-93462-predictor namespace: kserve-ci-e2e-test resourceVersion: "26331" uid: 9e8cf8e2-1c1d-4f69-81c6-30df6446be01 kind: Event lastTimestamp: "2026-07-16T22:23:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:22:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:22Z" name: success-200-isvc-93462-predictor.18c2e558e6af22b7 namespace: kserve-ci-e2e-test resourceVersion: "26980" uid: 4e38968f-ecee-4bfc-a3b0-fd7f1252b2dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:23:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-93462-predictor namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 9e8cf8e2-1c1d-4f69-81c6-30df6446be01 kind: Event lastTimestamp: "2026-07-16T22:24:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:07Z" name: success-200-isvc-93462-predictor.18c2e55c65785528 namespace: kserve-ci-e2e-test resourceVersion: "27429" uid: b65e4004-183d-41b4-8c74-d879c77a3bd7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-16T22:23:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-93462-predictor namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 9e8cf8e2-1c1d-4f69-81c6-30df6446be01 kind: Event lastTimestamp: "2026-07-16T22:24:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:07Z" name: success-200-isvc-93462-predictor.18c2e55c65e66023 namespace: kserve-ci-e2e-test resourceVersion: "27430" uid: 8fcb6e68-3b50-4ac8-a9b7-6d44682a3c56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-93462 namespace: kserve-ci-e2e-test resourceVersion: "26310" uid: 0447468d-1bef-4af2-944d-995c47e06d05 kind: Event lastTimestamp: "2026-07-16T22:23:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:23:35Z" name: success-200-isvc-93462.18c2e55567f75984 namespace: kserve-ci-e2e-test resourceVersion: "27118" uid: c69d9668-71c1-40bc-9ebb-f49254ec6d80 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-93462 namespace: kserve-ci-e2e-test resourceVersion: "26310" uid: 0447468d-1bef-4af2-944d-995c47e06d05 kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: 'Failed to update status for InferenceService "success-200-isvc-93462": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-93462": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:22:37Z" name: success-200-isvc-93462.18c2e55568eae33b namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: c26297a4-e93c-4ba2-8ea6-eea23f52f6bd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-93462 namespace: kserve-ci-e2e-test resourceVersion: "26310" uid: 0447468d-1bef-4af2-944d-995c47e06d05 kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-93462": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:22:37Z" name: success-200-isvc-93462.18c2e55568eb370e namespace: kserve-ci-e2e-test resourceVersion: "26342" uid: ef6fb01d-24c6-45c8-ae38-4f342f7a443a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-93462 namespace: kserve-ci-e2e-test resourceVersion: "27116" uid: 0447468d-1bef-4af2-944d-995c47e06d05 kind: Event lastTimestamp: "2026-07-16T22:23:35Z" message: InferenceService [success-200-isvc-93462] is Ready metadata: creationTimestamp: "2026-07-16T22:23:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:23:35Z" name: success-200-isvc-93462.18c2e562e42f98b5 namespace: kserve-ci-e2e-test resourceVersion: "27117" uid: 3251fdc9-3040-453c-9e7b-09b840e53281 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:13:12.477132Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7 namespace: kserve-ci-e2e-test resourceVersion: "21693" uid: 6de4c344-fd98-4d3d-a0d8-04cdd26a394a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a3a91-predictor-d4f677b68-tggk7 to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:13:12Z" name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7.18c2e4d1e1a54b5d namespace: kserve-ci-e2e-test resourceVersion: "21699" uid: 990680f4-34e7-49e2-990d-3a272d84c9ea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 6de4c344-fd98-4d3d-a0d8-04cdd26a394a kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a3a91-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:12Z" name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7.18c2e4d1ea42022e namespace: kserve-ci-e2e-test resourceVersion: "21730" uid: 2a753888-6512-42f9-a840-e91087b8ed6e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7 namespace: kserve-ci-e2e-test resourceVersion: "21704" uid: 6de4c344-fd98-4d3d-a0d8-04cdd26a394a kind: Event lastTimestamp: "2026-07-16T22:13:13Z" message: Add eth0 [10.134.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:13:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:13:13Z" name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7.18c2e4d22071180c namespace: kserve-ci-e2e-test resourceVersion: "21754" uid: 9688e1a0-e551-4dfe-916c-5c2d95ff38df reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 6de4c344-fd98-4d3d-a0d8-04cdd26a394a kind: Event lastTimestamp: "2026-07-16T22:13:13Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:13:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:13Z" name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7.18c2e4d221ef5b23 namespace: kserve-ci-e2e-test resourceVersion: "21756" uid: 0e97ee79-3cf5-4597-9b92-26a0f2a47c36 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 6de4c344-fd98-4d3d-a0d8-04cdd26a394a kind: Event lastTimestamp: "2026-07-16T22:13:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:13:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:13Z" name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7.18c2e4d2250ca0e7 namespace: kserve-ci-e2e-test resourceVersion: "21758" uid: 9d7b4f71-0f95-46a7-a07c-3468cb8d31cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 6de4c344-fd98-4d3d-a0d8-04cdd26a394a kind: Event lastTimestamp: "2026-07-16T22:13:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:13:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:13Z" name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7.18c2e4d225b08914 namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: a2228235-d971-4bf8-92fb-5f78859e1717 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 6de4c344-fd98-4d3d-a0d8-04cdd26a394a kind: Event lastTimestamp: "2026-07-16T22:13:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:13:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:13Z" name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7.18c2e4d225bd106a namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: 9c69cfd9-8f41-42fc-8557-2408ee49b756 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 6de4c344-fd98-4d3d-a0d8-04cdd26a394a kind: Event lastTimestamp: "2026-07-16T22:13:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:13:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:13Z" name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7.18c2e4d2295e453e namespace: kserve-ci-e2e-test resourceVersion: "21761" uid: a5ed0612-1da4-4119-8ffd-51adc942394f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 6de4c344-fd98-4d3d-a0d8-04cdd26a394a kind: Event lastTimestamp: "2026-07-16T22:13:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:13:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:13:13Z" name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7.18c2e4d229dd32e9 namespace: kserve-ci-e2e-test resourceVersion: "21762" uid: fee7b652-6299-4622-945b-89b92c00f022 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-16T22:13:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 6de4c344-fd98-4d3d-a0d8-04cdd26a394a kind: Event lastTimestamp: "2026-07-16T22:22:41Z" message: 'Readiness probe failed: dial tcp 10.134.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:41Z" name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7.18c2e4d2824b22e0 namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 1bb6e834-252b-4944-badc-e3334132187f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 6de4c344-fd98-4d3d-a0d8-04cdd26a394a kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:37Z" name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7.18c2e555683bb6ad namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: d6dc6691-e7d7-4810-9c66-2f89e558b3a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 6de4c344-fd98-4d3d-a0d8-04cdd26a394a kind: Event lastTimestamp: "2026-07-16T22:22:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:37Z" name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7.18c2e555683c5265 namespace: kserve-ci-e2e-test resourceVersion: "26338" uid: 63ea06c8-ab1e-449b-b086-0af0b18b335b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 6de4c344-fd98-4d3d-a0d8-04cdd26a394a kind: Event lastTimestamp: "2026-07-16T22:22:41Z" message: 'Readiness probe failed: Get "https://10.134.0.51:8643/healthz": dial tcp 10.134.0.51:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:22:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:41Z" name: success-200-isvc-a3a91-predictor-d4f677b68-tggk7.18c2e5564abfd8fd namespace: kserve-ci-e2e-test resourceVersion: "26460" uid: 494d0eaf-b739-4723-9aef-78bc21e25ec0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a3a91-predictor-d4f677b68 namespace: kserve-ci-e2e-test resourceVersion: "21683" uid: c4cc4df3-ab56-4348-8cf8-715de4004c48 kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: 'Created pod: success-200-isvc-a3a91-predictor-d4f677b68-tggk7' metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:12Z" name: success-200-isvc-a3a91-predictor-d4f677b68.18c2e4d1e102f0ba namespace: kserve-ci-e2e-test resourceVersion: "21695" uid: 4cbd8380-9e6d-436f-b92d-5b9592025902 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a3a91-predictor namespace: kserve-ci-e2e-test resourceVersion: "21681" uid: 3c5be137-7296-400a-aaca-57148494175b kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: Scaled up replica set success-200-isvc-a3a91-predictor-d4f677b68 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:12Z" name: success-200-isvc-a3a91-predictor.18c2e4d1dedefacf namespace: kserve-ci-e2e-test resourceVersion: "21685" uid: b251e31a-eb48-4287-b16d-e28d6e3b3ef1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:13:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3a91-predictor namespace: kserve-ci-e2e-test resourceVersion: "21682" uid: 2acb7c31-b9aa-4b25-869c-a02dc1225778 kind: Event lastTimestamp: "2026-07-16T22:14:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:13:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:27Z" name: success-200-isvc-a3a91-predictor.18c2e4d55d763833 namespace: kserve-ci-e2e-test resourceVersion: "22473" uid: 928a6549-6aa9-4c2c-b545-d14a334f896e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:13:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3a91-predictor namespace: kserve-ci-e2e-test resourceVersion: "21682" uid: 2acb7c31-b9aa-4b25-869c-a02dc1225778 kind: Event lastTimestamp: "2026-07-16T22:14:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:13:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:14:27Z" name: success-200-isvc-a3a91-predictor.18c2e4d55df0c2ba namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: d7f4bcf4-b4f1-4711-a7d2-3d0de80b1496 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: 5a1d752c-7e91-41ff-9b6e-0541554d2551 kind: Event lastTimestamp: "2026-07-16T22:14:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:14:11Z" name: success-200-isvc-a3a91.18c2e4d1dee8168c namespace: kserve-ci-e2e-test resourceVersion: "22315" uid: 2b25f2d4-8dbf-4e77-a25f-b766a093459c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: 5a1d752c-7e91-41ff-9b6e-0541554d2551 kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-a3a91": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a3a91": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:13:12Z" name: success-200-isvc-a3a91.18c2e4d1df931202 namespace: kserve-ci-e2e-test resourceVersion: "21690" uid: 23fe3c7e-b9b3-4cea-8036-33620aac8933 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: 5a1d752c-7e91-41ff-9b6e-0541554d2551 kind: Event lastTimestamp: "2026-07-16T22:13:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a3a91": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:13:12Z" name: success-200-isvc-a3a91.18c2e4d1df936a37 namespace: kserve-ci-e2e-test resourceVersion: "21691" uid: 21e20721-7602-4c4a-8510-c24667eaec2a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:14:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3a91 namespace: kserve-ci-e2e-test resourceVersion: "22312" uid: 5a1d752c-7e91-41ff-9b6e-0541554d2551 kind: Event lastTimestamp: "2026-07-16T22:14:11Z" message: InferenceService [success-200-isvc-a3a91] is Ready metadata: creationTimestamp: "2026-07-16T22:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:14:11Z" name: success-200-isvc-a3a91.18c2e4df91cdc74c namespace: kserve-ci-e2e-test resourceVersion: "22313" uid: 5ba4da29-f4ae-4c82-a611-6e5bf0e943fe reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:22:01.497417Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9 namespace: kserve-ci-e2e-test resourceVersion: "25918" uid: 6a885f6f-87ef-4ca3-8830-d272bb0a0127 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9 to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:22:01Z" name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9.18c2e54d0db63699 namespace: kserve-ci-e2e-test resourceVersion: "25935" uid: 06b6b87a-f2ed-41b2-ada6-a7208edfca64 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9 namespace: kserve-ci-e2e-test resourceVersion: "25946" uid: 6a885f6f-87ef-4ca3-8830-d272bb0a0127 kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: Add eth0 [10.134.0.53/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:22:01Z" name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9.18c2e54d28fbd569 namespace: kserve-ci-e2e-test resourceVersion: "25977" uid: 3192aa1c-4c37-4145-ae72-17c2dcb1c52b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 6a885f6f-87ef-4ca3-8830-d272bb0a0127 kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:01Z" name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9.18c2e54d2a9876c9 namespace: kserve-ci-e2e-test resourceVersion: "25979" uid: 9e6d6169-76bc-4257-941d-37790365561c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 6a885f6f-87ef-4ca3-8830-d272bb0a0127 kind: Event lastTimestamp: "2026-07-16T22:22:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:02Z" name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9.18c2e54d2e5d71c4 namespace: kserve-ci-e2e-test resourceVersion: "25980" uid: 7022f64c-4085-4c8c-9390-63bb501738e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 6a885f6f-87ef-4ca3-8830-d272bb0a0127 kind: Event lastTimestamp: "2026-07-16T22:22:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:02Z" name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9.18c2e54d2ee5feff namespace: kserve-ci-e2e-test resourceVersion: "25981" uid: 8dc3b15d-b604-4022-96d8-7bc801c24120 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 6a885f6f-87ef-4ca3-8830-d272bb0a0127 kind: Event lastTimestamp: "2026-07-16T22:22:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:02Z" name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9.18c2e54d2ef31387 namespace: kserve-ci-e2e-test resourceVersion: "25982" uid: 1567e304-a2f0-453a-bc3b-7a9443c9c828 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 6a885f6f-87ef-4ca3-8830-d272bb0a0127 kind: Event lastTimestamp: "2026-07-16T22:22:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:02Z" name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9.18c2e54d32a01221 namespace: kserve-ci-e2e-test resourceVersion: "25986" uid: 2b1dd375-97b9-488d-9fa5-2ebc6ed90d0c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 6a885f6f-87ef-4ca3-8830-d272bb0a0127 kind: Event lastTimestamp: "2026-07-16T22:22:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:02Z" name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9.18c2e54d332538e0 namespace: kserve-ci-e2e-test resourceVersion: "25987" uid: 8e3b4b26-43c1-42cd-bf85-54562aeec77f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T22:22:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 6a885f6f-87ef-4ca3-8830-d272bb0a0127 kind: Event lastTimestamp: "2026-07-16T22:22:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.53:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:22:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:49Z" name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9.18c2e54da120eeba namespace: kserve-ci-e2e-test resourceVersion: "26546" uid: 637b27f4-5a9a-44d9-af49-c064def1ebc6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 6a885f6f-87ef-4ca3-8830-d272bb0a0127 kind: Event lastTimestamp: "2026-07-16T22:23:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:21Z" name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9.18c2e55fc2e8bf41 namespace: kserve-ci-e2e-test resourceVersion: "26901" uid: 85944d9a-9a1a-478c-ae1f-deca30bc6265 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 6a885f6f-87ef-4ca3-8830-d272bb0a0127 kind: Event lastTimestamp: "2026-07-16T22:23:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:23:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:21Z" name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9.18c2e55fc2e938bb namespace: kserve-ci-e2e-test resourceVersion: "26904" uid: 64549ec2-7d20-4a04-981a-c90e5aef9241 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 6a885f6f-87ef-4ca3-8830-d272bb0a0127 kind: Event lastTimestamp: "2026-07-16T22:23:24Z" message: 'Readiness probe failed: Get "https://10.134.0.53:8643/healthz": dial tcp 10.134.0.53:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:23:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:24Z" name: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9.18c2e5607d66c2d4 namespace: kserve-ci-e2e-test resourceVersion: "27026" uid: 9c345809-860e-452b-a688-160a65fe3df4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a3d07-predictor-546c97cfb4 namespace: kserve-ci-e2e-test resourceVersion: "25905" uid: 8025ff06-1d65-44ec-88ca-27f658c02503 kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: 'Created pod: success-200-isvc-a3d07-predictor-546c97cfb4-9wvg9' metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:22:01Z" name: success-200-isvc-a3d07-predictor-546c97cfb4.18c2e54d0ca1622e namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: 52ca1d39-f3b2-4a19-aa19-a39cc8584591 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a3d07-predictor namespace: kserve-ci-e2e-test resourceVersion: "25900" uid: 2819b724-a441-49c8-9498-bf3012dc4574 kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: Scaled up replica set success-200-isvc-a3d07-predictor-546c97cfb4 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:22:01Z" name: success-200-isvc-a3d07-predictor.18c2e54d072c5730 namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 6b972900-d9c4-4e1a-9e05-9ecc4f0993c0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:22:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3d07-predictor namespace: kserve-ci-e2e-test resourceVersion: "25904" uid: 36431def-3e4a-4e32-8717-da0c60459608 kind: Event lastTimestamp: "2026-07-16T22:22:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:22:31Z" name: success-200-isvc-a3d07-predictor.18c2e550869fcc4a namespace: kserve-ci-e2e-test resourceVersion: "26244" uid: f08406bc-ae43-46bc-9c17-e2f8152099f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:22:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3d07-predictor namespace: kserve-ci-e2e-test resourceVersion: "25904" uid: 36431def-3e4a-4e32-8717-da0c60459608 kind: Event lastTimestamp: "2026-07-16T22:22:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:22:31Z" name: success-200-isvc-a3d07-predictor.18c2e55087867115 namespace: kserve-ci-e2e-test resourceVersion: "26246" uid: a70e2c0d-9758-41c2-91f1-1d0ce7d227e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:22:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3d07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: 36431def-3e4a-4e32-8717-da0c60459608 kind: Event lastTimestamp: "2026-07-16T22:23:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:16Z" name: success-200-isvc-a3d07-predictor.18c2e55785c63b30 namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: 4c0f0397-6d6e-4ecf-88d1-b9649576fc1c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:22:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3d07-predictor namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: 36431def-3e4a-4e32-8717-da0c60459608 kind: Event lastTimestamp: "2026-07-16T22:23:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:16Z" name: success-200-isvc-a3d07-predictor.18c2e557861b8488 namespace: kserve-ci-e2e-test resourceVersion: "26814" uid: eb01840c-3fb0-4a7d-a2d5-0339ff5a7e47 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3d07 namespace: kserve-ci-e2e-test resourceVersion: "25883" uid: 7392f426-b5d8-485f-9f8d-db31a566dd50 kind: Event lastTimestamp: "2026-07-16T22:23:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:23:00Z" name: success-200-isvc-a3d07.18c2e54d072da1a0 namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 917cf94f-b084-4d7e-8b2b-79df2a314082 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3d07 namespace: kserve-ci-e2e-test resourceVersion: "25883" uid: 7392f426-b5d8-485f-9f8d-db31a566dd50 kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: 'Failed to update status for InferenceService "success-200-isvc-a3d07": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a3d07": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:22:01Z" name: success-200-isvc-a3d07.18c2e54d07e20436 namespace: kserve-ci-e2e-test resourceVersion: "25912" uid: 44ef1340-39e0-4df0-9ca1-0d38d36b8c7f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3d07 namespace: kserve-ci-e2e-test resourceVersion: "25883" uid: 7392f426-b5d8-485f-9f8d-db31a566dd50 kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a3d07": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:22:01Z" name: success-200-isvc-a3d07.18c2e54d07e251a2 namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: 31b92a56-a0df-4c8a-9aa3-d54761977458 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3d07 namespace: kserve-ci-e2e-test resourceVersion: "26641" uid: 7392f426-b5d8-485f-9f8d-db31a566dd50 kind: Event lastTimestamp: "2026-07-16T22:23:00Z" message: InferenceService [success-200-isvc-a3d07] is Ready metadata: creationTimestamp: "2026-07-16T22:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:23:00Z" name: success-200-isvc-a3d07.18c2e55ab3399814 namespace: kserve-ci-e2e-test resourceVersion: "26642" uid: 1b29b849-1d23-4f21-bde5-82515841e868 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:12:36.525975Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 56149faf-9310-48bd-b667-2f9e9216898e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:12:36Z" name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr.18c2e4c982ca289d namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: 25c4849d-2c10-40f7-9b54-828f54aa6e51 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr namespace: kserve-ci-e2e-test resourceVersion: "21212" uid: 56149faf-9310-48bd-b667-2f9e9216898e kind: Event lastTimestamp: "2026-07-16T22:12:36Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:12:36Z" name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr.18c2e4c99e04ad91 namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: fa313313-5515-43c8-b768-139a3e1d486e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 56149faf-9310-48bd-b667-2f9e9216898e kind: Event lastTimestamp: "2026-07-16T22:12:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1745" already present on machine metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:37Z" name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr.18c2e4c99fe54190 namespace: kserve-ci-e2e-test resourceVersion: "21256" uid: 0df665f6-017e-40a9-af01-e42af26645fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 56149faf-9310-48bd-b667-2f9e9216898e kind: Event lastTimestamp: "2026-07-16T22:12:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:37Z" name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr.18c2e4c9a3ebea57 namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: b936b9b9-67d5-474d-a7fc-61fcff0dcf9b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 56149faf-9310-48bd-b667-2f9e9216898e kind: Event lastTimestamp: "2026-07-16T22:12:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:37Z" name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr.18c2e4c9a484f103 namespace: kserve-ci-e2e-test resourceVersion: "21258" uid: 4b99b640-a78a-447a-9a46-ebdca13bea40 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 56149faf-9310-48bd-b667-2f9e9216898e kind: Event lastTimestamp: "2026-07-16T22:12:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:37Z" name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr.18c2e4c9a494e492 namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: a0f07696-65cd-4138-88ff-db51fd93be68 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 56149faf-9310-48bd-b667-2f9e9216898e kind: Event lastTimestamp: "2026-07-16T22:12:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:37Z" name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr.18c2e4c9a89ada32 namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: f010ca89-de82-4a68-9889-26a9d3103d98 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 56149faf-9310-48bd-b667-2f9e9216898e kind: Event lastTimestamp: "2026-07-16T22:12:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:12:37Z" name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr.18c2e4c9a91a9104 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: c1505c33-ec45-48d5-9d5d-2c738606f32f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-16T22:12:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 56149faf-9310-48bd-b667-2f9e9216898e kind: Event lastTimestamp: "2026-07-16T22:22:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:12:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:04Z" name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr.18c2e4ca13bef7a1 namespace: kserve-ci-e2e-test resourceVersion: "26027" uid: 387ab949-cf00-456c-a86e-c5dc4da2ceb0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 56149faf-9310-48bd-b667-2f9e9216898e kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:01Z" name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr.18c2e54d07024e3b namespace: kserve-ci-e2e-test resourceVersion: "25907" uid: 90262369-1f14-4456-9fd1-5091173de146 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 56149faf-9310-48bd-b667-2f9e9216898e kind: Event lastTimestamp: "2026-07-16T22:22:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:01Z" name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr.18c2e54d0702fa81 namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: 79982840-6dfb-456d-acef-5bd3687eef74 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:22:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 56149faf-9310-48bd-b667-2f9e9216898e kind: Event lastTimestamp: "2026-07-16T22:22:04Z" message: 'Readiness probe failed: Get "https://10.134.0.49:8643/healthz": dial tcp 10.134.0.49:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:22:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:22:04Z" name: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr.18c2e54ddc4601c5 namespace: kserve-ci-e2e-test resourceVersion: "26026" uid: 8bd4a5aa-880c-4717-a319-82c3a989034c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c1850-predictor-7f7fd6c8cb namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: bacf2caa-0dd2-4756-9053-f0732a31bb1e kind: Event lastTimestamp: "2026-07-16T22:12:36Z" message: 'Created pod: success-200-isvc-c1850-predictor-7f7fd6c8cb-z78wr' metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:36Z" name: success-200-isvc-c1850-predictor-7f7fd6c8cb.18c2e4c98242eec1 namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: b2e3d517-d34d-4251-809c-ab8a891c4af8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c1850-predictor namespace: kserve-ci-e2e-test resourceVersion: "21177" uid: 25d8805a-7b6b-4762-a058-c111bfbb555d kind: Event lastTimestamp: "2026-07-16T22:12:36Z" message: Scaled up replica set success-200-isvc-c1850-predictor-7f7fd6c8cb from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:12:36Z" name: success-200-isvc-c1850-predictor.18c2e4c98047e636 namespace: kserve-ci-e2e-test resourceVersion: "21181" uid: dc109f86-c39e-43d0-ba99-588f80c530e9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:12:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1850-predictor namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: 0dd22437-f5b1-46c5-a11d-204bd10c79a8 kind: Event lastTimestamp: "2026-07-16T22:13:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:06Z" name: success-200-isvc-c1850-predictor.18c2e4ccff40f5ab namespace: kserve-ci-e2e-test resourceVersion: "21573" uid: abb984a7-0959-47cf-876c-9d7dd7a3b944 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:12:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1850-predictor namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: 0dd22437-f5b1-46c5-a11d-204bd10c79a8 kind: Event lastTimestamp: "2026-07-16T22:13:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:06Z" name: success-200-isvc-c1850-predictor.18c2e4ccffb84b6e namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: e355dd49-f7e4-4e2c-842a-e51d46b9bd32 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1850-predictor namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: 0dd22437-f5b1-46c5-a11d-204bd10c79a8 kind: Event lastTimestamp: "2026-07-16T22:13:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:51Z" name: success-200-isvc-c1850-predictor.18c2e4d3fded7311 namespace: kserve-ci-e2e-test resourceVersion: "22146" uid: 8dd7223e-dd4e-45b5-9050-cb46ded00a26 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1850-predictor namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: 0dd22437-f5b1-46c5-a11d-204bd10c79a8 kind: Event lastTimestamp: "2026-07-16T22:13:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:13:51Z" name: success-200-isvc-c1850-predictor.18c2e4d3fed7dbf6 namespace: kserve-ci-e2e-test resourceVersion: "22147" uid: c3122646-58c4-49b2-969d-f0c7192059ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:12:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c1850 namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 2b35dfbc-668a-4084-a92f-51d0345f551e kind: Event lastTimestamp: "2026-07-16T22:13:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:13:35Z" name: success-200-isvc-c1850.18c2e4c980bcc459 namespace: kserve-ci-e2e-test resourceVersion: "21979" uid: c94fe4e1-c9f0-4608-a6ee-d6e986d8cce5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:13:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c1850 namespace: kserve-ci-e2e-test resourceVersion: "21977" uid: 2b35dfbc-668a-4084-a92f-51d0345f551e kind: Event lastTimestamp: "2026-07-16T22:13:35Z" message: InferenceService [success-200-isvc-c1850] is Ready metadata: creationTimestamp: "2026-07-16T22:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:13:35Z" name: success-200-isvc-c1850.18c2e4d723690ae7 namespace: kserve-ci-e2e-test resourceVersion: "21978" uid: db34ca99-b9bf-4e89-907c-553de1a392d8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:00:16.735177Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq to ip-10-0-129-202.ec2.internal metadata: creationTimestamp: "2026-07-16T22:00:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:00:16Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e41d43d2697b namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 41be713b-547a-4bba-85e5-f45f0edc2d32 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d9a7f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:17Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e41d53c45e80 namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: d0e7dc31-2418-4eee-b254-1a506fc047b0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14631" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:00:17Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e41d82946733 namespace: kserve-ci-e2e-test resourceVersion: "14769" uid: ce76812e-a243-4f7b-b9aa-0a8a69492da7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:00:17Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1745" metadata: creationTimestamp: "2026-07-16T22:00:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:17Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e41d842a44f9 namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 46553316-af55-4e7b-8b04-70906d5b7fe1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:00:33Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1745" in 15.568s (15.568s including waiting). Image size: 1335607397 bytes.' metadata: creationTimestamp: "2026-07-16T22:00:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:33Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e421241bd141 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: fad0c872-0deb-4392-ae16-86ab5da5d950 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:00:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-16T22:00:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:33Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e421287419bc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 912d25ce-fc5c-4b44-abcb-f6dd0c32dbdd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:00:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-16T22:00:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:33Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e421291fad6f namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: 85a05c88-9c7a-4714-b8d4-2877524e0ab3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:00:33Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-16T22:00:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:33Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e421292f88c7 namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: 117452da-68ef-4af7-b25d-14e694cc7ffa reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:00:35Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.501s (2.501s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-16T22:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:35Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e421be4cc6ce namespace: kserve-ci-e2e-test resourceVersion: "14970" uid: 7d80b44e-16ab-4190-a861-bab64fb1f590 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:00:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-16T22:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:36Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e421c259dabf namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: c0d1d746-a71e-4fd3-81fc-264572e9a0e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:00:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:00:36Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e421c2dc99cc namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: ebe53446-f6aa-4583-a7cd-77524f186d39 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-16T22:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:01:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:52Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e421c945f848 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 87bf991d-f86a-4fb9-aa2e-580d931131cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e43346441507 namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: ee667b04-9961-4d9a-ae1a-194f3889ac14 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:51Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e43346447edb namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 7961732b-1e9b-4118-8766-1470be70ddbf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: d1f09258-3f0e-4e16-8890-939e6a4d7c90 kind: Event lastTimestamp: "2026-07-16T22:01:52Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-16T22:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:52Z" name: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq.18c2e4337b748293 namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: 5c2dffc5-2758-463e-964d-0a84490665ea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-202.ec2.internal source: component: kubelet host: ip-10-0-129-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d9a7f-predictor-956dc5df6 namespace: kserve-ci-e2e-test resourceVersion: "14615" uid: f812c9ea-83f2-4858-8e55-64c5f49281e7 kind: Event lastTimestamp: "2026-07-16T22:00:16Z" message: 'Created pod: success-200-isvc-d9a7f-predictor-956dc5df6-ckrlq' metadata: creationTimestamp: "2026-07-16T22:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:00:16Z" name: success-200-isvc-d9a7f-predictor-956dc5df6.18c2e41d437217bf namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: bf73c874-acda-4abc-9e84-77fff3ea4337 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d9a7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: e6555be4-3f22-4513-884a-78ea0a1e9f2b kind: Event lastTimestamp: "2026-07-16T22:00:16Z" message: Scaled up replica set success-200-isvc-d9a7f-predictor-956dc5df6 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:00:16Z" name: success-200-isvc-d9a7f-predictor.18c2e41d3fbcceee namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: 321fb5f4-79ce-4c40-82b4-dac0258e1ba1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:00:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d9a7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: 56593bb1-3ea7-4815-b8e3-d5654a68b284 kind: Event lastTimestamp: "2026-07-16T22:01:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:00:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:01Z" name: success-200-isvc-d9a7f-predictor.18c2e420c554a94d namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: 54b88c8b-cc3b-4b4f-892f-3e8f4d9c53ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:00:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d9a7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: 56593bb1-3ea7-4815-b8e3-d5654a68b284 kind: Event lastTimestamp: "2026-07-16T22:01:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:00:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:01Z" name: success-200-isvc-d9a7f-predictor.18c2e420c5d0c576 namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: e115412c-1381-418a-b35f-37cd10a50302 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:01:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d9a7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: 56593bb1-3ea7-4815-b8e3-d5654a68b284 kind: Event lastTimestamp: "2026-07-16T22:01:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:46Z" name: success-200-isvc-d9a7f-predictor.18c2e42b42aec873 namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 2b737ac2-147a-4454-822c-4d32130f565e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-16T22:01:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d9a7f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: 56593bb1-3ea7-4815-b8e3-d5654a68b284 kind: Event lastTimestamp: "2026-07-16T22:01:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:46Z" name: success-200-isvc-d9a7f-predictor.18c2e42b432249bf namespace: kserve-ci-e2e-test resourceVersion: "15637" uid: 70f35c4d-db11-4e03-960d-0320ff02ef0a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-16T22:00:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d9a7f namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: eb38b57e-191c-42f6-8061-9c6fb7b2aa54 kind: Event lastTimestamp: "2026-07-16T22:01:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-16T22:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:01:22Z" name: success-200-isvc-d9a7f.18c2e41d45e7ffc8 namespace: kserve-ci-e2e-test resourceVersion: "15395" uid: d70ee1b6-2b96-4f46-91e0-804005f4e4d1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d9a7f namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: eb38b57e-191c-42f6-8061-9c6fb7b2aa54 kind: Event lastTimestamp: "2026-07-16T22:00:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-d9a7f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d9a7f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:00:16Z" name: success-200-isvc-d9a7f.18c2e41d46692103 namespace: kserve-ci-e2e-test resourceVersion: "14633" uid: 8360f8fc-41c3-4089-9169-85dd315ad00a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:00:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d9a7f namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: eb38b57e-191c-42f6-8061-9c6fb7b2aa54 kind: Event lastTimestamp: "2026-07-16T22:00:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d9a7f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-16T22:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:00:16Z" name: success-200-isvc-d9a7f.18c2e41d466984ae namespace: kserve-ci-e2e-test resourceVersion: "14635" uid: 86556f8f-5117-4b1f-be11-39b954cb85eb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d9a7f namespace: kserve-ci-e2e-test resourceVersion: "15393" uid: eb38b57e-191c-42f6-8061-9c6fb7b2aa54 kind: Event lastTimestamp: "2026-07-16T22:01:22Z" message: InferenceService [success-200-isvc-d9a7f] is Ready metadata: creationTimestamp: "2026-07-16T22:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:01:22Z" name: success-200-isvc-d9a7f.18c2e42c87fcdcfb namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: 60b03ff7-f4e3-4ca3-a467-ee8c334c8626 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:03:01.381291Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-69517-c7754998c-6lqsn namespace: kserve-ci-e2e-test resourceVersion: "16656" uid: 9ae74149-4ea3-4866-82a2-3682c63e516c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-69517-c7754998c-6lqsn to ip-10-0-131-231.ec2.internal metadata: creationTimestamp: "2026-07-16T22:03:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:03:01Z" name: switch-graph-69517-c7754998c-6lqsn.18c2e443997eb694 namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: f83fd149-949d-4265-b33d-dd1400f60e02 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:01Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-69517-c7754998c-6lqsn namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: 9ae74149-4ea3-4866-82a2-3682c63e516c kind: Event lastTimestamp: "2026-07-16T22:03:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-69517-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:03:01Z" name: switch-graph-69517-c7754998c-6lqsn.18c2e443a562dbc2 namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: 5e68a8a0-303b-484c-aa7f-f95cbe589950 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:02Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-69517-c7754998c-6lqsn namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: 9ae74149-4ea3-4866-82a2-3682c63e516c kind: Event lastTimestamp: "2026-07-16T22:03:02Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:03:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:03:02Z" name: switch-graph-69517-c7754998c-6lqsn.18c2e443d69bff47 namespace: kserve-ci-e2e-test resourceVersion: "16683" uid: 1d29dbfd-a127-473b-b3d1-3ab2d4d6bc1b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-69517} kind: Pod name: switch-graph-69517-c7754998c-6lqsn namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: 9ae74149-4ea3-4866-82a2-3682c63e516c kind: Event lastTimestamp: "2026-07-16T22:03:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7e35538adc84c05d9420d87001e9355aeddffd6326bb391010f8e6c1a3c85203" already present on machine metadata: creationTimestamp: "2026-07-16T22:03:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:03:02Z" name: switch-graph-69517-c7754998c-6lqsn.18c2e443d80b7b0d namespace: kserve-ci-e2e-test resourceVersion: "16686" uid: 32ab3bc8-f798-458d-b2ad-3b66f3da9bc9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-69517} kind: Pod name: switch-graph-69517-c7754998c-6lqsn namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: 9ae74149-4ea3-4866-82a2-3682c63e516c kind: Event lastTimestamp: "2026-07-16T22:03:02Z" message: 'Created container: switch-graph-69517' metadata: creationTimestamp: "2026-07-16T22:03:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:03:02Z" name: switch-graph-69517-c7754998c-6lqsn.18c2e443dab1fbd9 namespace: kserve-ci-e2e-test resourceVersion: "16687" uid: ae86f5b9-5cbc-4bca-b65b-867629526d7b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-69517} kind: Pod name: switch-graph-69517-c7754998c-6lqsn namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: 9ae74149-4ea3-4866-82a2-3682c63e516c kind: Event lastTimestamp: "2026-07-16T22:03:02Z" message: Started container switch-graph-69517 metadata: creationTimestamp: "2026-07-16T22:03:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:03:02Z" name: switch-graph-69517-c7754998c-6lqsn.18c2e443db173f13 namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 189ad5a4-91b5-4ec6-aae2-a9b7c601a832 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:11:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-69517} kind: Pod name: switch-graph-69517-c7754998c-6lqsn namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: 9ae74149-4ea3-4866-82a2-3682c63e516c kind: Event lastTimestamp: "2026-07-16T22:11:15Z" message: Stopping container switch-graph-69517 metadata: creationTimestamp: "2026-07-16T22:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:15Z" name: switch-graph-69517-c7754998c-6lqsn.18c2e4b6bf7d204f namespace: kserve-ci-e2e-test resourceVersion: "20171" uid: c8f7db74-12e6-4b2d-958f-08230d538d3b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:11:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-69517} kind: Pod name: switch-graph-69517-c7754998c-6lqsn namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: 9ae74149-4ea3-4866-82a2-3682c63e516c kind: Event lastTimestamp: "2026-07-16T22:11:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T22:11:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:11:44Z" name: switch-graph-69517-c7754998c-6lqsn.18c2e4b789425519 namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 4f18bb54-f1f2-46e7-b4d1-6c642e13ffb0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-69517-c7754998c namespace: kserve-ci-e2e-test resourceVersion: "16653" uid: 368c4213-7f24-429d-8556-5ec12d996c01 kind: Event lastTimestamp: "2026-07-16T22:03:01Z" message: 'Created pod: switch-graph-69517-c7754998c-6lqsn' metadata: creationTimestamp: "2026-07-16T22:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:01Z" name: switch-graph-69517-c7754998c.18c2e443990dc841 namespace: kserve-ci-e2e-test resourceVersion: "16658" uid: fc14e8e6-ad2a-4946-b2c7-15c0e9413654 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-69517 namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 75ab1235-ba60-4340-b887-fdb8c8cea865 kind: Event lastTimestamp: "2026-07-16T22:03:01Z" message: Scaled up replica set switch-graph-69517-c7754998c from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:01Z" name: switch-graph-69517.18c2e44397d6f6a0 namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: 405e34c1-2759-433d-9e1e-292d056bd632 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-69517 namespace: kserve-ci-e2e-test resourceVersion: "16668" uid: 67c4cf0f-4536-4e30-ad77-1c43a3996db7 kind: Event lastTimestamp: "2026-07-16T22:03:01Z" message: InferenceGraph [switch-graph-69517] is Ready metadata: creationTimestamp: "2026-07-16T22:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:03:01Z" name: switch-graph-69517.18c2e4439e8a83de namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: 0df8d6f4-72b0-498c-b848-88c1cb19b50c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-69517 namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: ab5ceef9-91bc-4e26-b70b-4e3f3d07b2fe kind: Event lastTimestamp: "2026-07-16T22:03:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:03:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:16Z" name: switch-graph-69517.18c2e447167fa4e8 namespace: kserve-ci-e2e-test resourceVersion: "16810" uid: b1bb9ca5-16a5-44bd-818a-47102480ba86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-69517 namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: ab5ceef9-91bc-4e26-b70b-4e3f3d07b2fe kind: Event lastTimestamp: "2026-07-16T22:03:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:03:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:16Z" name: switch-graph-69517.18c2e44716d65b5a namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: df3c5d41-4243-47bf-89f7-f4b395dc3554 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-69517 namespace: kserve-ci-e2e-test resourceVersion: "16811" uid: ab5ceef9-91bc-4e26-b70b-4e3f3d07b2fe kind: Event lastTimestamp: "2026-07-16T22:03:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:03:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:31Z" name: switch-graph-69517.18c2e44a95f0756f namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: 7192f10c-b2b8-481c-a67e-2ee58d54cb3a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:03:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-69517 namespace: kserve-ci-e2e-test resourceVersion: "16811" uid: ab5ceef9-91bc-4e26-b70b-4e3f3d07b2fe kind: Event lastTimestamp: "2026-07-16T22:03:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-16T22:03:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:03:31Z" name: switch-graph-69517.18c2e44a967706a3 namespace: kserve-ci-e2e-test resourceVersion: "16930" uid: 542f735e-0588-4b77-92bf-19c937830837 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:23:47.523463Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-93462-6584468b49-vtflc namespace: kserve-ci-e2e-test resourceVersion: "27243" uid: 5b417db4-3fb1-4246-b106-5113a3334c64 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-93462-6584468b49-vtflc to ip-10-0-131-231.ec2.internal metadata: creationTimestamp: "2026-07-16T22:23:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:23:47Z" name: switch-graph-93462-6584468b49-vtflc.18c2e565bd5b3786 namespace: kserve-ci-e2e-test resourceVersion: "27249" uid: 43a77dd4-7698-44e5-8dd0-62821873fb5e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:47Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-93462-6584468b49-vtflc namespace: kserve-ci-e2e-test resourceVersion: "27251" uid: 5b417db4-3fb1-4246-b106-5113a3334c64 kind: Event lastTimestamp: "2026-07-16T22:23:47Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:23:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:23:47Z" name: switch-graph-93462-6584468b49-vtflc.18c2e565d644105f namespace: kserve-ci-e2e-test resourceVersion: "27260" uid: 1ca28c4e-552c-465d-a89d-4b48e556ddf4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-93462} kind: Pod name: switch-graph-93462-6584468b49-vtflc namespace: kserve-ci-e2e-test resourceVersion: "27244" uid: 5b417db4-3fb1-4246-b106-5113a3334c64 kind: Event lastTimestamp: "2026-07-16T22:23:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7e35538adc84c05d9420d87001e9355aeddffd6326bb391010f8e6c1a3c85203" already present on machine metadata: creationTimestamp: "2026-07-16T22:23:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:47Z" name: switch-graph-93462-6584468b49-vtflc.18c2e565d7bb0589 namespace: kserve-ci-e2e-test resourceVersion: "27262" uid: f994e392-df2d-4fb8-8835-3afac14e0bbc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-93462} kind: Pod name: switch-graph-93462-6584468b49-vtflc namespace: kserve-ci-e2e-test resourceVersion: "27244" uid: 5b417db4-3fb1-4246-b106-5113a3334c64 kind: Event lastTimestamp: "2026-07-16T22:23:48Z" message: 'Created container: switch-graph-93462' metadata: creationTimestamp: "2026-07-16T22:23:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:48Z" name: switch-graph-93462-6584468b49-vtflc.18c2e565daba4353 namespace: kserve-ci-e2e-test resourceVersion: "27263" uid: 9a42043c-bf3f-49c0-aa35-e6aa020f59d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-93462} kind: Pod name: switch-graph-93462-6584468b49-vtflc namespace: kserve-ci-e2e-test resourceVersion: "27244" uid: 5b417db4-3fb1-4246-b106-5113a3334c64 kind: Event lastTimestamp: "2026-07-16T22:23:48Z" message: Started container switch-graph-93462 metadata: creationTimestamp: "2026-07-16T22:23:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:23:48Z" name: switch-graph-93462-6584468b49-vtflc.18c2e565db2f133a namespace: kserve-ci-e2e-test resourceVersion: "27264" uid: 50d5c033-caa7-4ac1-8ae2-a530de78e25c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:40:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-93462} kind: Pod name: switch-graph-93462-6584468b49-vtflc namespace: kserve-ci-e2e-test resourceVersion: "27244" uid: 5b417db4-3fb1-4246-b106-5113a3334c64 kind: Event lastTimestamp: "2026-07-16T22:40:06Z" message: Stopping container switch-graph-93462 metadata: creationTimestamp: "2026-07-16T22:40:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:40:06Z" name: switch-graph-93462-6584468b49-vtflc.18c2e649c5be9e14 namespace: kserve-ci-e2e-test resourceVersion: "34144" uid: 631433c6-d0bf-4a60-b5cb-aa9fa9a8fac2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:40:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-93462} kind: Pod name: switch-graph-93462-6584468b49-vtflc namespace: kserve-ci-e2e-test resourceVersion: "27244" uid: 5b417db4-3fb1-4246-b106-5113a3334c64 kind: Event lastTimestamp: "2026-07-16T22:40:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T22:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:40:34Z" name: switch-graph-93462-6584468b49-vtflc.18c2e64a7646d4a4 namespace: kserve-ci-e2e-test resourceVersion: "34421" uid: a685248a-c071-4cd3-890e-ee4872b66ada reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-93462-6584468b49 namespace: kserve-ci-e2e-test resourceVersion: "27239" uid: 193b43f6-b3b1-41ad-a8f8-035bf56fd1d3 kind: Event lastTimestamp: "2026-07-16T22:23:47Z" message: 'Created pod: switch-graph-93462-6584468b49-vtflc' metadata: creationTimestamp: "2026-07-16T22:23:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:47Z" name: switch-graph-93462-6584468b49.18c2e565bc1f5d1b namespace: kserve-ci-e2e-test resourceVersion: "27246" uid: fb084663-7712-4c8c-a6f9-2503842166b5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-93462 namespace: kserve-ci-e2e-test resourceVersion: "27236" uid: 6b916765-8f79-4b4d-8955-1afae9dde2e1 kind: Event lastTimestamp: "2026-07-16T22:23:47Z" message: Scaled up replica set switch-graph-93462-6584468b49 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:23:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:23:47Z" name: switch-graph-93462.18c2e565b9e179c0 namespace: kserve-ci-e2e-test resourceVersion: "27240" uid: 20c89fe4-51d8-4c56-8691-a00b5b8dc21a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:23:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-93462 namespace: kserve-ci-e2e-test resourceVersion: "27253" uid: f0128558-a9d1-4212-8d6d-08b518ebcc97 kind: Event lastTimestamp: "2026-07-16T22:23:47Z" message: InferenceGraph [switch-graph-93462] is Ready metadata: creationTimestamp: "2026-07-16T22:23:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:23:47Z" name: switch-graph-93462.18c2e565c154ba8d namespace: kserve-ci-e2e-test resourceVersion: "27255" uid: e21eab5b-c255-40c5-a0ec-72d9d2a29568 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:24:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-93462 namespace: kserve-ci-e2e-test resourceVersion: "27237" uid: 4f04bfa4-82af-4136-a729-b7c889622005 kind: Event lastTimestamp: "2026-07-16T22:24:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:17Z" name: switch-graph-93462.18c2e56939da3cce namespace: kserve-ci-e2e-test resourceVersion: "27510" uid: ebdae153-1989-4991-81ad-e2d5069cd992 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-16T22:24:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-93462 namespace: kserve-ci-e2e-test resourceVersion: "27237" uid: 4f04bfa4-82af-4136-a729-b7c889622005 kind: Event lastTimestamp: "2026-07-16T22:24:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-16T22:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:24:17Z" name: switch-graph-93462.18c2e5693a6cd10e namespace: kserve-ci-e2e-test resourceVersion: "27511" uid: c4a20186-a81c-4093-abd5-b767fd333f1c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-16T22:01:36.860807Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d9a7f-56f7cf8887-dnrnd namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 2767c720-5514-4c08-8943-641933c19a7e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d9a7f-56f7cf8887-dnrnd to ip-10-0-131-231.ec2.internal metadata: creationTimestamp: "2026-07-16T22:01:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-16T22:01:36Z" name: switch-graph-d9a7f-56f7cf8887-dnrnd.18c2e42febae8ac4 namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: e34744b0-d376-48e1-a9f4-d94de3dde811 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d8c48f66c-rphj2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:37Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d9a7f-56f7cf8887-dnrnd namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 2767c720-5514-4c08-8943-641933c19a7e kind: Event lastTimestamp: "2026-07-16T22:01:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-d9a7f-serving-cert" not found' metadata: creationTimestamp: "2026-07-16T22:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:37Z" name: switch-graph-d9a7f-56f7cf8887-dnrnd.18c2e42ff51175a7 namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 5422f299-1769-4618-a03f-05b84a0f1552 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:37Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d9a7f-56f7cf8887-dnrnd namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: 2767c720-5514-4c08-8943-641933c19a7e kind: Event lastTimestamp: "2026-07-16T22:01:37Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-16T22:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-16T22:01:37Z" name: switch-graph-d9a7f-56f7cf8887-dnrnd.18c2e4302930163b namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: fc394fb2-0035-4ef5-b6da-daefda1ee6f0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d9a7f} kind: Pod name: switch-graph-d9a7f-56f7cf8887-dnrnd namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 2767c720-5514-4c08-8943-641933c19a7e kind: Event lastTimestamp: "2026-07-16T22:01:37Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:7e35538adc84c05d9420d87001e9355aeddffd6326bb391010f8e6c1a3c85203" metadata: creationTimestamp: "2026-07-16T22:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:37Z" name: switch-graph-d9a7f-56f7cf8887-dnrnd.18c2e4302ac502a1 namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: c16874e5-64fc-4528-972e-9e2352bbd06d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d9a7f} kind: Pod name: switch-graph-d9a7f-56f7cf8887-dnrnd namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 2767c720-5514-4c08-8943-641933c19a7e kind: Event lastTimestamp: "2026-07-16T22:01:40Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:7e35538adc84c05d9420d87001e9355aeddffd6326bb391010f8e6c1a3c85203" in 2.478s (2.478s including waiting). Image size: 218081378 bytes.' metadata: creationTimestamp: "2026-07-16T22:01:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:40Z" name: switch-graph-d9a7f-56f7cf8887-dnrnd.18c2e430be8387fa namespace: kserve-ci-e2e-test resourceVersion: "15585" uid: 5e892823-ffc7-4df0-a40d-609b96bc9f64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d9a7f} kind: Pod name: switch-graph-d9a7f-56f7cf8887-dnrnd namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 2767c720-5514-4c08-8943-641933c19a7e kind: Event lastTimestamp: "2026-07-16T22:01:40Z" message: 'Created container: switch-graph-d9a7f' metadata: creationTimestamp: "2026-07-16T22:01:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:40Z" name: switch-graph-d9a7f-56f7cf8887-dnrnd.18c2e430c1c68b07 namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: fdf8ef64-4d5d-41b6-853f-750331f632d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d9a7f} kind: Pod name: switch-graph-d9a7f-56f7cf8887-dnrnd namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 2767c720-5514-4c08-8943-641933c19a7e kind: Event lastTimestamp: "2026-07-16T22:01:40Z" message: Started container switch-graph-d9a7f metadata: creationTimestamp: "2026-07-16T22:01:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:40Z" name: switch-graph-d9a7f-56f7cf8887-dnrnd.18c2e430c23ed5ba namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 05eb339c-8eda-4d31-8731-9c392ca1f9bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d9a7f} kind: Pod name: switch-graph-d9a7f-56f7cf8887-dnrnd namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 2767c720-5514-4c08-8943-641933c19a7e kind: Event lastTimestamp: "2026-07-16T22:01:51Z" message: Stopping container switch-graph-d9a7f metadata: creationTimestamp: "2026-07-16T22:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:01:51Z" name: switch-graph-d9a7f-56f7cf8887-dnrnd.18c2e4333c12fd42 namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: ab3b8df8-7580-4146-a340-dbdfc2014964 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-16T22:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d9a7f} kind: Pod name: switch-graph-d9a7f-56f7cf8887-dnrnd namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 2767c720-5514-4c08-8943-641933c19a7e kind: Event lastTimestamp: "2026-07-16T22:02:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-16T22:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-16T22:02:17Z" name: switch-graph-d9a7f-56f7cf8887-dnrnd.18c2e433734df211 namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 883aea6e-d27d-459b-8a95-b1b38aba779e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-231.ec2.internal source: component: kubelet host: ip-10-0-131-231.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d9a7f-56f7cf8887 namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: ef6a4c82-df44-4567-9b29-1cbe52212187 kind: Event lastTimestamp: "2026-07-16T22:01:36Z" message: 'Created pod: switch-graph-d9a7f-56f7cf8887-dnrnd' metadata: creationTimestamp: "2026-07-16T22:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:36Z" name: switch-graph-d9a7f-56f7cf8887.18c2e42feaff1ff9 namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: 9c619abe-8f94-462b-9ef5-a65cb6b1c889 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d9a7f namespace: kserve-ci-e2e-test resourceVersion: "15535" uid: ca306a6e-d70e-48b6-a56a-a85e8689d713 kind: Event lastTimestamp: "2026-07-16T22:01:36Z" message: Scaled up replica set switch-graph-d9a7f-56f7cf8887 from 0 to 1 metadata: creationTimestamp: "2026-07-16T22:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-16T22:01:36Z" name: switch-graph-d9a7f.18c2e42fe9efc4a6 namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 494f0edc-cf2e-48ba-a7cf-54137f74a828 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-16T22:01:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d9a7f namespace: kserve-ci-e2e-test resourceVersion: "15551" uid: 5df3c440-8e0a-419a-9dfd-32a0f6d103ed kind: Event lastTimestamp: "2026-07-16T22:01:36Z" message: InferenceGraph [switch-graph-d9a7f] is Ready metadata: creationTimestamp: "2026-07-16T22:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-16T22:01:36Z" name: switch-graph-d9a7f.18c2e42ff01f4edb namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: b90100b2-0f4b-40bf-be42-820c0ba27be0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "34427"